This repository has been archived by the owner on Aug 25, 2024. It is now read-only.

feature: git: github api: Enable create or update logic from title, b… #5001

Workflow file for this run

.github/workflows/testing.yml at 144cc4b

	name: Tests

	on:
	workflow_dispatch: null
	push:
	branches:
	- main
	paths-ignore:
	- 'docs/arch/**'
	pull_request:
	paths-ignore:
	- 'docs/arch/**'

	jobs:
	build:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- "3.12"
	steps:
	- uses: actions/checkout@v2
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Get pip cache
	id: pip-cache
	run: \|
	python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
	- name: pip cache
	uses: actions/cache@v1
	with:
	path: ${{ steps.pip-cache.outputs.dir }}
	key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.cfg') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dev dependencies
	run: \|
	pip install -U pip setuptools wheel build
	pip install -U sbom4python
	pip install -U https://github.com/scitt-community/scitt-api-emulator/archive/e89a60584fa717382f279ae24b8a1a93d458bb4d.zip
	pip install -e .[dev]
	python -m pip freeze
	- name: Build
	run: \|
	python -m build .
	- name: Generate SBOM
	id: generate-sbom
	uses: anthonyharrison/sbom4python@5b458354df89357bf0253e62ea4567b1807120e2
	with:
	python-version: ${{ matrix.python-version }}
	module-name: dffml
	output-directory: sbom
	- name: in-toto attestation for cyclonedx SBOM
	id: in-toto-cyclonedx
	env:
	MODULE_NAME: dffml
	run: \|
	echo "attestation<<GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT
	(python -m json.tool --sort-keys \| tee -a $GITHUB_OUTPUT) <<EOF
	{
	"_type": "https://in-toto.io/Statement/v0.1",
	"subject": [
	{
	"name": "$(cd dist/ && echo *.tar.gz)",
	"digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.tar.gz) \| awk '{print $1}')"}
	},
	{
	"name": "$(cd dist/ && echo *.whl)",
	"digest": {"sha256": "$(cd dist/ && sha256sum $(echo *.whl) \| awk '{print $1}')"}
	}
	],
	"predicateType": "https://cyclonedx.org/bom/v1.4",
	"predicate": $(cat "${MODULE_NAME}-py${{ matrix.python-version }}.json")
	}
	EOF
	echo "GITHUB_OUTPUT_EOF" >> $GITHUB_OUTPUT
	- name: Checkout public-keys branch
	uses: actions/checkout@v4
	with:
	ref: public-keys
	path: public-keys
	- name: Generate keypair to sign SCITT statement
	id: scitt-gen-keypair
	run: \|
	ssh-keygen -q -f ssh-private -t ecdsa -b 384 -N '' -C "$(head -n 100 /dev/urandom \| sha384sum \| awk '{print $1}')" -I "$(date -Iseconds)" <<<y
	cat ssh-private \| python -c 'import sys; from cryptography.hazmat.primitives import serialization; print(serialization.load_ssh_private_key(sys.stdin.buffer.read(), password=None).private_bytes(encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption()).decode().rstrip())' > private-key.pem
	# cat ssh-private.pub \| tee -a public-keys/x509
	- name: SSH key as x509
	shell: python -u {0}
	run: \|
	from cryptography import x509
	from cryptography.x509.oid import NameOID
	from cryptography.hazmat.primitives import hashes
	from cryptography.hazmat.backends import default_backend
	from cryptography.hazmat.primitives.asymmetric import rsa
	from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
	from jwcrypto import jwk, jws
	import datetime
	import pathlib

	key = jwk.JWK.from_pem(pathlib.Path("private-key.pem").read_bytes())
	rsa_public_key = jwk.JWK.from_json(key.export_public())

	# Convert the JWK to a public key
	public_key = rsa_public_key.get_op_key('verify')

	# Create a builder for the X.509 certificate
	subject = issuer = x509.Name([
	x509.NameAttribute(NameOID.COUNTRY_NAME, "US"),
	x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Oregon"),
	x509.NameAttribute(NameOID.LOCALITY_NAME, "Portland"),
	x509.NameAttribute(NameOID.ORGANIZATION_NAME, "SCITT Emulator"),
	x509.NameAttribute(NameOID.COMMON_NAME, "example.com"),
	])

	cert_builder = x509.CertificateBuilder(
	subject_name=subject,
	issuer_name=issuer,
	public_key=public_key,
	serial_number=x509.random_serial_number(),
	not_valid_before=datetime.datetime.utcnow(),
	not_valid_after=datetime.datetime.utcnow() + datetime.timedelta(days=1), # Certificate valid for 1 day
	extensions=[]
	)

	# Self-sign the certificate with the private key
	private_key_op = key.get_op_key('sign')
	cert = cert_builder.sign(private_key=private_key_op, algorithm=hashes.SHA256(), backend=default_backend())

	# Serialize the certificate
	cert_pem = cert.public_bytes(encoding=Encoding.PEM)

	# Display or save the PEM encoded certificate
	contents = b""
	try:
	pathlib.Path("public-keys", "x509").read_bytes()
	except:
	pass
	pathlib.Path("public-keys", "x509").write_bytes(contents + b"\n" + cert_pem)
	- name: Remove ssh private
	run: \|
	rm -v ssh-private
	- name: Push new public key
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	set -xe
	cd public-keys
	gh auth setup-git
	git config --global --add safe.directory "${PWD}"
	git config --global user.email "[email protected]"
	git config --global user.name "GitHub Actions"
	git add -A
	# If no delta clean exit
	git commit -sm "Snapshot" \|\| exit 0
	git push -uf origin "HEAD:public-keys"
	# Wait for propagation
	# TODO x509 style polling?
	sleep 10s
	- name: Submit SBOM to SCITT
	id: scitt-submit-sbom
	uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654
	with:
	issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509
	subject: pkg:github/${{ github.repository }}@${{ github.sha }}
	payload: ${{ steps.in-toto-cyclonedx.outputs.attestation }}
	private-key-pem: private-key.pem
	scitt-url: https://scitt.unstable.chadig.com
	- name: Create Pull Request
	if: ${{ steps.generate-sbom.outputs.changed }}
	uses: peter-evans/[email protected]
	with:
	commit-message: "chore: update SBOM for Python ${{ matrix.python-version }}"
	title: "chore: update SBOM for Python ${{ matrix.python-version }}"
	branch: chore-sbom-py${{ matrix.python-version }}
	delete-branch: true
	author: GitHub Actions <[email protected]>
	add-paths: sbom
	- uses: openvex/generate-vex@159b7ee4845fb48f1991395ce8501d6263407360
	name: Run vexctl
	id: vexctl
	with:
	product: pkg:github/${{ github.repository }}@${{ github.sha }}
	- name: Submit OpenVEX to SCITT
	id: scitt-submit-openvex
	uses: scitt-community/scitt-api-emulator@f1f5c16630a28511e970b6903fbc4c0db6c07654
	with:
	issuer: did:web:raw.githubusercontent.com:intel:dffml:public-keys:x509
	subject: pkg:github/${{ github.repository }}@${{ github.sha }}
	payload: ${{ steps.vexctl.outputs.openvex }}
	private-key-pem: private-key.pem
	scitt-url: https://scitt.unstable.chadig.com
	- name: Remove private key used in keypair to sign SCITT statement
	run: \|
	rm -v private-key.pem

	lint:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	max-parallel: 40
	matrix:
	check: [changelog, whitespace, commit, lines]
	python-version: [3.7]
	node-version: [12.x]

	steps:
	- uses: actions/checkout@v2
	- name: Checkout full upstream repo
	run: \|
	git remote set-url origin https://github.com/intel/dffml
	git fetch --prune --unshallow
	git fetch --depth=1 origin +refs/tags/:refs/tags/
	git config --global user.email "[email protected]"
	git config --global user.name "DFFML CI/CD"
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Use Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v1
	with:
	node-version: ${{ matrix.node-version }}
	- name: Get pip cache
	id: pip-cache
	run: \|
	python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
	- name: pip cache
	uses: actions/cache@v1
	with:
	path: ${{ steps.pip-cache.outputs.dir }}
	key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|
	set -x
	./.ci/deps.sh ${{ matrix.check }}
	- name: Run check
	run: \|
	export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
	SSH_DFFML_GH_PAGES=${{ secrets.SSH_DFFML_GH_PAGES }} ./.ci/run.sh ${{ matrix.check }}

	container:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v2
	- name: Install dependencies
	run: \|
	set -x
	curl -fsSL https://download.docker.com/linux/ubuntu/gpg \| sudo apt-key add -
	sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
	sudo apt-get update && sudo apt-get install -y docker-ce docker-ce-cli containerd.io
	- name: Run check
	run: \|
	./.ci/run.sh container

	test:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	max-parallel: 100
	matrix:
	plugin:
	- model/daal4py
	- model/tensorflow
	- model/scratch
	- model/scikit
	- model/vowpalWabbit
	- model/autosklearn
	- model/xgboost
	- operations/binsec
	- operations/data
	- operations/deploy
	- operations/image
	- operations/nlp
	- source/mysql
	- feature/git
	- feature/auth
	- service/http
	- configloader/yaml
	- configloader/image
	- source/mongodb
	- entities/alice
	python-version:
	- "3.12"

	steps:
	- uses: actions/checkout@v2
	- name: Checkout full upstream repo
	run: \|
	git remote set-url origin https://github.com/intel/dffml
	git fetch --prune --unshallow
	git fetch --depth=1 origin +refs/tags/:refs/tags/
	git config --global user.email "[email protected]"
	git config --global user.name "DFFML CI/CD"
	- name: Remove unused software
	run: \|
	sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Get pip cache
	id: pip-cache
	run: \|
	python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
	- name: pip cache
	uses: actions/cache@v1
	with:
	path: ${{ steps.pip-cache.outputs.dir }}
	key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: shouldi test binary cache
	uses: actions/cache@v1
	with:
	path: examples/shouldi/tests/downloads
	key: ${{ hashFiles('examples/shouldi/tests/binaries.py') }}
	restore-keys: \|
	${{ runner.os }}-shouldi-test-binaries-
	- name: Install dependencies
	run: \|
	set -x
	./.ci/deps.sh ${{ matrix.plugin }}
	- name: Test
	run: \|
	set -x
	export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
	export PYTHON=python${{ matrix.python-version }}
	export PLUGIN=${{ matrix.plugin }}
	export LOGGING=debug
	export PATH="${HOME}/.local/bin:${PATH}"
	export PYPI_TOKENS=$(mktemp)
	cat > ${PYPI_TOKENS} <<EOF
	examples/shouldi=${{ secrets.PYPI_SHOULDI }}
	model/daal4py=${{ secrets.PYPI_MODEL_DAAL4PY }}
	model/tensorflow=${{ secrets.PYPI_MODEL_TENSORFLOW }}
	model/tensorflow_hub=${{ secrets.PYPI_MODEL_TENSORFLOW_HUB }}
	model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }}
	model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }}
	model/spacy=${{ secrets.PYPI_MODEL_SPACY }}
	model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }}
	model/autosklearn=${{ secrets.PYPI_MODEL_AUTOSKLEARN }}
	model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }}
	model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }}
	source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }}
	source/mongodb=${{ secrets.PYPI_SOURCE_MONGODB }}
	feature/git=${{ secrets.PYPI_FEATURE_GIT }}
	feature/auth=${{ secrets.PYPI_FEATURE_AUTH }}
	operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }}
	operations/data=${{ secrets.PYPI_OPERATIONS_DATA }}
	operations/deploy=${{ secrets.PYPI_OPERATIONS_DEPLOY }}
	operations/image=${{ secrets.PYPI_OPERATIONS_IMAGE }}
	operations/nlp=${{ secrets.PYPI_OPERATIONS_NLP }}
	service/http=${{ secrets.PYPI_SERVICE_HTTP }}
	configloader/yaml=${{ secrets.PYPI_CONFIG_YAML }}
	configloader/image=${{ secrets.PYPI_CONFIG_IMAGE }}
	EOF
	export TWINE_USERNAME=__token__
	if [ "x${PLUGIN}" = "x." ]; then
	export TWINE_PASSWORD=${{ secrets.PYPI_DFFML }}
	else
	export TWINE_PASSWORD=$(grep "${PLUGIN}=" "${PYPI_TOKENS}" \| sed 's/^[^=]*=//g')
	fi
	rm ${PYPI_TOKENS}
	./.ci/run.sh "${PLUGIN}"
	if [ "x${PLUGIN}" = "x." ]; then
	CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} codecov
	fi

	tutorials:
	runs-on: ubuntu-latest
	if: false
	strategy:
	fail-fast: false
	max-parallel: 100
	matrix:
	python-version: [3.7]
	docs:
	- docs/cli.rst
	- docs/examples/dataflows.rst
	- docs/examples/integration.rst
	- docs/examples/or_covid_data_by_county.rst
	- docs/examples/shouldi.rst
	- docs/examples/innersource/swportal.rst
	- docs/examples/innersource/microservice.rst
	- docs/examples/innersource/kubernetes.rst
	- docs/examples/icecream_sales.rst
	- docs/examples/data_cleanup/data_cleanup.rst
	- docs/examples/data_cleanup/data_cleanup_classfication.rst
	- docs/installation.rst
	- docs/troubleshooting.rst
	- docs/tutorials/accuracy/mse.rst
	- docs/tutorials/dataflows/io.rst
	- docs/tutorials/dataflows/nlp.rst
	- docs/tutorials/models/iris.rst
	- docs/tutorials/models/package.rst
	- docs/tutorials/models/docs.rst
	- docs/tutorials/models/slr.rst
	- docs/tutorials/sources/complex.rst
	- docs/tutorials/sources/file.rst

	steps:
	- uses: actions/checkout@v2
	- name: Checkout full upstream repo
	run: \|
	git remote set-url origin https://github.com/intel/dffml
	git fetch --prune --unshallow
	git fetch --depth=1 origin +refs/tags/:refs/tags/
	git config --global user.email "[email protected]"
	git config --global user.name "DFFML CI/CD"
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Get pip cache
	id: pip-cache
	run: \|
	python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
	- name: pip cache
	uses: actions/cache@v1
	with:
	path: ${{ steps.pip-cache.outputs.dir }}
	key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|
	set -x
	./.ci/deps.sh .
	- name: Test
	run: \|
	set -x
	export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
	export PYTHON=python${{ matrix.python-version }}
	export PLUGIN=${{ matrix.plugin }}
	export LOGGING=debug
	export PATH="${HOME}/.local/bin:${PATH}"
	./.ci/run.sh consoletest ${{ matrix.docs }}

	macos:
	runs-on: macos-latest
	if: false
	strategy:
	fail-fast: false
	matrix:
	python-version: [3.7]
	steps:
	- uses: actions/checkout@v2
	- name: Checkout full upstream repo
	run: \|
	git remote set-url origin https://github.com/intel/dffml
	git fetch --prune --unshallow
	git config --global user.email "[email protected]"
	git config --global user.name "DFFML CI/CD"
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v2
	with:
	python-version: ${{ matrix.python-version }}
	- name: Get pip cache
	id: pip-cache
	run: \|
	python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)"
	- name: pip cache
	uses: actions/cache@v1
	with:
	path: ${{ steps.pip-cache.outputs.dir }}
	key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Setup DFFML
	run: \|
	pip install -U pip setuptools wheel
	pip install -e .[dev]
	dffml service dev install -skip model/daal4py
	# XGBoost requires libomp on OSX
	brew install libomp
	- name: Test
	run: \|
	python -m unittest discover -v
	python -m pip freeze

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feature: git: github api: Enable create or update logic from title, b… #5001

Workflow file

feature: git: github api: Enable create or update logic from title, b… #5001

Jobs

Run details

Workflow file for this run