From c58eaa8d39f6cb06f9c41f1496dde5f84820a46f Mon Sep 17 00:00:00 2001 From: adplincinst Date: Wed, 29 May 2024 17:16:34 +0000 Subject: [PATCH 1/4] added second harvest to gleaner gh action to verify idempotency. Warning current test may fail if test starts on a given day but finishes next day as prov json-ld documents will increase due to embedded datestamp causes creation of new json-ld file due to content-addressing scheme --- .../workflows/test_geoconnex_dev_sitemap.yml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_geoconnex_dev_sitemap.yml b/.github/workflows/test_geoconnex_dev_sitemap.yml index cd2aafa3..58bf2949 100644 --- a/.github/workflows/test_geoconnex_dev_sitemap.yml +++ b/.github/workflows/test_geoconnex_dev_sitemap.yml @@ -45,4 +45,26 @@ jobs: sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refgages0 330 sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refmainstems 66 echo "::endgroup::" - + - name: Build and Run Gleaner Harvest #2 + run: | + echo "::group::Build and Run Gleaner Harvest" + make + sh scripts/iow/start-gleaner.sh + echo "::endgroup::" + - name: Verify Harvest Counts on #2 Harvest (Verify Idempotency) + run: | + echo "::group::Verify Harvest Counts on First Harvest (Pristine S3)" + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/orgs 5 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/cdss0 30 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/dams0 45 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/nmwdist0 266 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/refgages0 330 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/refmainstems 66 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/cdss0 30 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/dams0 45 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/nmwdist0 265 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refgages0 330 + sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refmainstems 66 + echo "::endgroup::" + + From f50fd4796fefb44b07e5f736f7938741bf829105 Mon Sep 17 00:00:00 2001 From: adplincinst Date: Wed, 29 May 2024 17:19:19 +0000 Subject: [PATCH 2/4] added pull_request trigger to test_geoconnex_dev_sitemap.yml --- .github/workflows/test_geoconnex_dev_sitemap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_geoconnex_dev_sitemap.yml b/.github/workflows/test_geoconnex_dev_sitemap.yml index 58bf2949..cd712ad5 100644 --- a/.github/workflows/test_geoconnex_dev_sitemap.yml +++ b/.github/workflows/test_geoconnex_dev_sitemap.yml @@ -1,7 +1,7 @@ name: Test Gleaner against pids.geoconnex.dev Sitemap on: - [ workflow_dispatch ] + [ workflow_dispatch, pull_request ] jobs: main: runs-on: ubuntu-22.04 From 9789f8c3b8d3a3880f08257403b99d78bffe4465 Mon Sep 17 00:00:00 2001 From: adplincinst Date: Wed, 29 May 2024 17:22:30 +0000 Subject: [PATCH 3/4] updated gh action grouping labels --- .github/workflows/test_geoconnex_dev_sitemap.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_geoconnex_dev_sitemap.yml b/.github/workflows/test_geoconnex_dev_sitemap.yml index cd712ad5..8337095d 100644 --- a/.github/workflows/test_geoconnex_dev_sitemap.yml +++ b/.github/workflows/test_geoconnex_dev_sitemap.yml @@ -47,13 +47,13 @@ jobs: echo "::endgroup::" - name: Build and Run Gleaner Harvest #2 run: | - echo "::group::Build and Run Gleaner Harvest" + echo "::group::Build and Run Gleaner Harvest Again" make sh scripts/iow/start-gleaner.sh echo "::endgroup::" - name: Verify Harvest Counts on #2 Harvest (Verify Idempotency) run: | - echo "::group::Verify Harvest Counts on First Harvest (Pristine S3)" + echo "::group::Verify Harvest Counts on Second Harvest (Verify Idempotency)" sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/orgs 5 sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/cdss0 30 sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/prov/dams0 45 From a8e16a1d1e90ad3bf2ad8ac2d3a46df77e65742b Mon Sep 17 00:00:00 2001 From: adplincinst Date: Wed, 29 May 2024 17:26:40 +0000 Subject: [PATCH 4/4] remove hash charcters because it causes truncation in labels during gh action --- .github/workflows/test_geoconnex_dev_sitemap.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_geoconnex_dev_sitemap.yml b/.github/workflows/test_geoconnex_dev_sitemap.yml index 8337095d..283a8557 100644 --- a/.github/workflows/test_geoconnex_dev_sitemap.yml +++ b/.github/workflows/test_geoconnex_dev_sitemap.yml @@ -45,13 +45,12 @@ jobs: sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refgages0 330 sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/summoned/refmainstems 66 echo "::endgroup::" - - name: Build and Run Gleaner Harvest #2 + - name: Run Gleaner Harvest Again run: | - echo "::group::Build and Run Gleaner Harvest Again" - make + echo "::group:: Run Gleaner Harvest Again" sh scripts/iow/start-gleaner.sh echo "::endgroup::" - - name: Verify Harvest Counts on #2 Harvest (Verify Idempotency) + - name: Verify Harvest Counts on Second Harvest (Verify Idempotency) run: | echo "::group::Verify Harvest Counts on Second Harvest (Verify Idempotency)" sh scripts/iow/assert-s3-dir-cnt.sh $HOME/bin/mc myminio/iow/orgs 5