From b443b70a828a34c79c71e41af9cb2eab7b950a14 Mon Sep 17 00:00:00 2001 From: Matthew Fernandez Date: Tue, 1 Dec 2020 07:02:30 -0800 Subject: [PATCH] rewrite deployment CI task to create releases on Gitlab MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Prior to this commit, the deployment CI task runs on a private runner and uploads release artifacts to www2.graphviz.org. The SSH steps in the deployment task are currently failing due to connectivity issues. The current maintainers are unable to fix this as we do not have enough knowledge of how this work flow was intended to function. Apart from these concerns, Gitlab have recently introduced support for so-called “generic packages” [0] to host binary artifacts directly on Gitlab (or its underlying CDN) as well as the ability to programmatically create releases [1] on the releases page of a project. The decision was made to simplify the Graphviz infrastructure and host future releases (both stable and development) on Gitlab. This change rewrites the deployment task to (1) run on the Gitlab shared runner using a release-cli Docker image, (2) upload package artifacts to Graphviz' generic package, and (3) create a release from this for stable version numbers. To comprehend the role the generic package container is playing here, note that generic package version numbers are synthesized from commit SHA and are not intended to be exposed to users. Users can browse the generic package page [2], but are only ever expected to be interested in the latest version available here. In contrast, version numbers on the release page [3] are intended to correspond to Graphviz stable version numbers. ci/deploy.py handles mapping a release on this page to its related (synthesized) generic package version. Closes #1892. [0]: https://docs.gitlab.com/ee/user/packages/generic_packages/index.html [1]: https://gitlab.com/gitlab-org/release-cli/-/blob/master/docs/index.md [2]: https://gitlab.com/graphviz/graphviz/-/packages [3]: https://gitlab.com/graphviz/graphviz/-/releases --- .gitlab-ci.yml | 29 +++------ DEVELOPERS.md | 19 ++---- ci/deploy.py | 169 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 ci/deploy.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 604511c6c..5ec328363 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -604,26 +604,15 @@ macos-cmake-test: deployment: stage: deploy + image: registry.gitlab.com/gitlab-org/release-cli:latest + # do not re-run this job for new Git tags of previously seen commits + except: + - tags + before_script: + - apk add --update-cache curl + - apk add --update-cache python3 script: - - GV_VERSION=$( cat VERSION ) - - COLLECTION=$( cat COLLECTION ) - - eval $(ssh-agent -s) - - cat "$DEPLOY_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan "$DEPLOY_HOST" >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - chmod -R o-rwx Packages - - chmod -R g-wx Packages - - chmod -R g+X Packages - - ssh "$DEPLOY_USER"@"$DEPLOY_HOST" 'rm -rf Packages/'"$COLLECTION"'/{fedora,centos,ubuntu,windows}' - - scp -r Packages/* "$DEPLOY_USER"@"$DEPLOY_HOST"':'Packages/ - - ssh "$DEPLOY_USER"@"$DEPLOY_HOST" 'for i in Packages/'"$COLLECTION"'/{fedora,centos}/*/{source,os/*,debug/*}; do createrepo $i; done' - - scp graphviz-fedora.repo graphviz-centos.repo "$DEPLOY_USER"@"$DEPLOY_HOST"':'Packages/ - - ssh "$DEPLOY_USER"@"$DEPLOY_HOST" mkdir -p Packages/"$COLLECTION"/portable_source - - md5sum graphviz-"$GV_VERSION".tar.gz >graphviz-"$GV_VERSION".tar.gz.md5 - - scp graphviz-"$GV_VERSION".tar.gz graphviz-"$GV_VERSION".tar.gz.md5 "$DEPLOY_USER"@"$DEPLOY_HOST"':'Packages/"$COLLECTION"/portable_source/ + - python3 ci/deploy.py --verbose + # do not run this job for MRs, developer’s forks, etc. only: - master@graphviz/graphviz - tags: - - graphviz,deploy diff --git a/DEVELOPERS.md b/DEVELOPERS.md index a83fd944e..addf9791a 100644 --- a/DEVELOPERS.md +++ b/DEVELOPERS.md @@ -86,19 +86,12 @@ is green [master pipeline](https://gitlab.com/graphviz/graphviz/-/pipelines?ref=master) to run for the new commit and check that it's green -1. Create a release at [GitHub releases](https://gitlab.com/graphviz/graphviz/-/releases) - - Fill in the `Tag name`, `Message` and `Release notes` fields. The - `Tag name` shall be a pure numeric new tag on the form - `..`. The `Message` field shall have the text - `Stable Release ..`. The `Release Notes` field - shall have the text `See the [CHANGELOG.md]()`. - - Example: - * **Tag name:** `2.44.1` - * **Message:** `Stable Release 2.44.1` - * **Release notes:** `See the [CHANGELOG](https://gitlab.com/graphviz/graphviz/-/blob/master/CHANGELOG.md#2441-2020-06-29).` +1. The “deployment” CI task will automatically create a release on the + [Gitlab releases tab](https://gitlab.com/graphviz/graphviz/-/releases). If a + release is not created, check that your modifications to `gen_version.py` + correctly set a version conforming to the regular expression `\d+\.\d+\.\d+`. + The “deployment” CI task will also create a Git tag for the version, e.g. + `2.44.1`. 1. Create a new local branch and name it e.g. `return-to--dev` diff --git a/ci/deploy.py b/ci/deploy.py new file mode 100644 index 000000000..27158ebb1 --- /dev/null +++ b/ci/deploy.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +''' +steps for deploying a new release (see ../.gitlab-ci.yml) + +This is based on Gitlab’s generic package example, +https://gitlab.com/gitlab-org/release-cli/-/tree/master/docs/examples/release-assets-as-generic-package +''' + +import argparse +import hashlib +import json +import logging +import os +import re +import shutil +import stat +import subprocess +import sys +from typing import Optional + +# logging output stream, setup in main() +log = None + +def upload(version: str, path: str, name: Optional[str] = None) -> str: + ''' + upload a file to the Graphviz generic package with the given version + ''' + + # use the path as the name if no other was given + if name is None: + name = path + + # Gitlab upload file_name field only allows letters, numbers, dot, dash, and + # underscore + safe = re.sub(r'[^a-zA-Z0-9.\-]', '_', name) + log.info(f'escaped name {name} to {safe}') + + target = f'{os.environ["CI_API_V4_URL"]}/projects/' \ + f'{os.environ["CI_PROJECT_ID"]}/packages/generic/graphviz-releases/' \ + f'{version}/{safe}' + + log.info(f'uploading {path} to {target}') + # calling Curl is not the cleanest way to achieve this, but Curl takes care of + # encodings, headers and part splitting for us + proc = subprocess.run(['curl', + '--silent', # no progress bar + '--include', # include HTTP response headers in output + '--verbose', # more connection details + '--retry', '3', # retry on transient errors + '--header', f'JOB-TOKEN: {os.environ["CI_JOB_TOKEN"]}', + '--upload-file', path, target], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, universal_newlines=True) + log.info('Curl response:') + for i, line in enumerate(proc.stdout.split('\n')): + log.info(f' {(i + 1):3}: {line}') + proc.check_returncode() + + resp = proc.stdout.split('\n')[-1] + if json.loads(resp)['message'] != '201 Created': + raise Exception(f'upload failed: {resp}') + + return target + +def main(args: [str]) -> int: + + # setup logging to print to stderr + global log + ch = logging.StreamHandler() + log = logging.getLogger('deploy.py') + log.addHandler(ch) + + # parse command line arguments + parser = argparse.ArgumentParser(description='Graphviz deployment script') + parser.add_argument('--version', help='Override version number used to ' + 'create a release. Without this, the contents of the VERSION file will be ' + 'used.') + parser.add_argument('--force', action='store_true', help='Force creating a ' + 'Gitlab release, even if the version number does not match \\d+.\\d+.\\d+.') + parser.add_argument('--verbose', action='store_true', help='Print more ' + 'diagnostic information.') + options = parser.parse_args(args[1:]) + + if options.verbose: + log.setLevel(logging.INFO) + + if os.environ.get('CI') is None: + log.error('CI environment variable unset; refusing to run') + return -1 + + # echo some useful things for debugging + log.info(f'os.uname(): {os.uname()}') + if os.path.exists('/etc/os-release'): + with open('/etc/os-release') as f: + log.info('/etc/os-release:') + for i, line in enumerate(f): + log.info(f' {i + 1}: {line[:-1]}') + + # bail out early if we do not have release-cli to avoid uploading assets that + # become orphaned when we fail to create the release + if not shutil.which('release-cli'): + log.error('release-cli not found') + return -1 + + # the generic package version has to be \d+.\d+.\d+ but it does not need to + # correspond to the release version (which may not conform to this pattern if + # this is a dev release), so generate a compliant generic package version + package_version = f'0.0.{int(os.environ["CI_COMMIT_SHA"], 16)}' + log.info(f'using generated generic package version {package_version}') + + # retrieve version name left by prior CI tasks + log.info('reading VERSION') + with open('VERSION') as f: + gv_version = f.read().strip() + log.info(f'VERSION == {gv_version}') + + tarball = f'graphviz-{gv_version}.tar.gz' + if not os.path.exists(tarball): + log.error(f'source {tarball} not found') + return -1 + + # generate a checksum for the source tarball + log.info(f'MD5 summing {tarball}') + checksum = f'{tarball}.md5' + with open(checksum, 'wt') as f: + with open(tarball, 'rb') as data: + f.write(f'{hashlib.md5(data.read()).hexdigest()} {tarball}\n') + + # list of assets we have uploaded + assets: [str] = [] + + assets.append(upload(package_version, tarball)) + assets.append(upload(package_version, checksum)) + + for stem, _, leaves in os.walk('Packages'): + for leaf in leaves: + path = os.path.join(stem, leaf) + + # get existing permissions + mode = os.stat(path).st_mode + + # fixup permissions, o-rwx g-wx + os.chmod(path, mode & ~stat.S_IRWXO & ~stat.S_IWGRP & ~stat.S_IXGRP) + + assets.append(upload(package_version, path, path[len('Packages/'):])) + + # we only create Gitlab releases for stable version numbers + if not options.force: + if re.match(r'\d+\.\d+\.\d+', options.version) is None: + log.warning(f'skipping release creation because {options.version} is not ' + 'of the form \\d+.\\d+.\\d+') + return 0 + + # construct a command to create the release itself + cmd = ['release-cli', 'create', '--name', options.version, '--tag-name', + options.version, '--description', 'See the [CHANGELOG](https://gitlab.com/' + 'graphviz/graphviz/-/blob/master/CHANGELOG.md).'] + for a in assets: + name = os.path.basename(a) + url = a + cmd += ['--assets-link', json.dumps({'name':name, 'url':url})] + + # create the release + subprocess.check_call(cmd) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) -- 2.40.0