{description}: {name}
\n".format( + **artifact) + html += "\n" + return html + +def create_github_status(report_url): + """ + Create a github status with a link to the report URL, + only do this once buildkite has been successful, so only report success here. + """ + status = repository.create_status( + COMMIT, + "success", + target_url=report_url, + description="KA-Lite Buildkite assets", + context="buildkite/kalite/assets" + ) + + if status: + logging.info("Successfully created GitHub status for commit {commit}.".format(commit=COMMIT)) + else: + logging.info("Error encountered. Now exiting!") + sys.exit(1) + +def collect_local_artifacts(): + """ + Create a list of artifacts + """ + + collected_artifacts = [] + + def create_artifact_data(artifact_dir): + for artifact in os.listdir(artifact_dir): + filename, file_extension = os.path.splitext(artifact) + file_extension = file_extension[1:] # Remove leading '.' + + if file_extension in file_manifest: + data = { + 'name': artifact, + 'file_location': "{artifact_dir}/{artifact}".format(artifact_dir=artifact_dir, artifact=artifact) + } + data.update(file_manifest[file_extension]) + logging.info("Collecting file data: {data}".format(data=data)) + collected_artifacts.append(data) + + create_artifact_data(DIST_DIR) + create_artifact_data(INSTALLER_DIR) + + return collected_artifacts + +def upload_artifacts(): + """ + Upload the artifacts to the Google Cloud Storage + Create a github status on the pull requester with the artifact media link. + """ + + client = storage.Client() + bucket = client.bucket("le-downloads") + artifacts = collect_local_artifacts() + is_release = os.getenv("IS_KALITE_RELEASE") + + for artifact in artifacts: + logging.info("Uploading file {filename}".format(filename=artifact.get("name"))) + + if is_release: + blob = bucket.blob("kalite/{release_dir}/{build_id}/{filename}".format( + release_dir=RELEASE_DIR, + build_id=BUILD_ID, + filename=artifact.get("name") + )) + else: + blob = bucket.blob("kalite/buildkite/build-{issue_id}/{build_id}/{filename}".format( + issue_id=ISSUE_ID, + build_id=BUILD_ID, + filename=artifact.get("name") + )) + + blob.upload_from_filename(filename=artifact.get("file_location")) + blob.make_public() + artifact.update({'media_url': blob.media_link}) + + html = create_status_report_html(artifacts) + blob = bucket.blob("kalite/{release_dir}/{build_id}/report.html".format(release_dir=RELEASE_DIR, build_id=BUILD_ID)) + blob.upload_from_string(html, content_type='text/html') + blob.make_public() + + logging.info("Status Report link: {}".format(blob.public_url)) + create_github_status(blob.public_url) + + if TAG: + # Building from a tag, this is probably a release! + get_release_asset_url = requests.get("https://api.github.com/repos/{owner}/{repo}/releases/tags/{tag}".format( + owner=REPO_OWNER, + repo=REPO_NAME, + tag=TAG + )) + + if get_release_asset_url.status_code == 200: + release_id = get_release_asset_url.json()['id'] + release_name = get_release_asset_url.json()['name'] + release = repository.release(id=release_id) + logging.info("Uploading build assets to GitHub Release: {release_name}".format(release_name=release_name)) + + for ext in file_order: + artifact_list = [] + for artifact_dict in artifacts: + if artifact_dict['extension'] == ext: + artifact_list.append(artifact_dict) + + for artifact in artifact_list: + logging.info("Uploading release asset: {artifact_name}".format(artifact.get('name'))) + # For some reason github3 does not let us set a label at initial upload + asset = release.upload_asset( + content_type=['content_type'], + name=artifact['name'], + asset=open(artifact['file_location'], 'rb') + ) + + if asset: + # So do it after the initial upload instead + asset.edit(artifact['name'], label=artifact['description']) + logging.info("Successfully uploaded release asset: {artifact}".format(artifact=artifact.get('name'))) + else: + logging.info("Error uploading release asset: {artifact}".format(artifact=artifact.get('name'))) + +def main(): + upload_artifacts() + +if __name__ == "__main__": + main() diff --git a/Dockerfile b/Dockerfile index 0f304b922c..0c7f985f66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,37 +5,25 @@ RUN apt-get -y update RUN apt-get install -y software-properties-common curl RUN add-apt-repository ppa:voronov84/andreyv RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - -RUN apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext python-sphinx wget - -# Install wine and related packages -RUN dpkg --add-architecture i386 -RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates sudo software-properties-common -RUN add-apt-repository -y ppa:ubuntu-wine/ppa && apt-get -y update && apt-get install --no-install-recommends --assume-yes wine - +RUN apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext wget COPY . /kalite VOLUME /kalitedist/ +# Use virtualenv's pip +ENV PIP=/kalite/kalite_env/bin/pip + # for mounting the whl files into other docker containers RUN pip install virtualenv && virtualenv /kalite/kalite_env --python=python2.7 -RUN /kalite/kalite_env/bin/pip install -r /kalite/requirements_dev.txt \ - && /kalite/kalite_env/bin/pip install -r /kalite/requirements_sphinx.txt \ - && /kalite/kalite_env/bin/pip install -e /kalite/. + +RUN $PIP install -r /kalite/requirements_dev.txt \ + && $PIP install -r /kalite/requirements_sphinx.txt \ + && $PIP install -e /kalite/. \ + && $PIP install pex # Override the PATH to add the path of our virtualenv python binaries first so it's python executes instead of # the system python. ENV PATH=/kalite/kalite_env/bin:$PATH ENV KALITE_PYTHON=/kalite/kalite_env/bin/python -# Installer dependencies -RUN cd /kalite/ && git clone https://github.com/learningequality/ka-lite-installers.git && cd /kalite/ka-lite-installers && git checkout 0.17.x -RUN cd /kalite/ka-lite-installers/windows && wget http://pantry.learningequality.org/downloads/ka-lite/0.17/content/contentpacks/en.zip - -# Build the python packages and the ka-lite windows installer -CMD cd /kalite && make dist \ - && cd /kalite/ka-lite-installers/windows \ - && cp -R /kalite/dist/ka_lite_static-*-py2-none-any.whl /kalite/ka-lite-installers/windows \ - && export KALITE_BUILD_VERSION=$(/kalite/kalite_env/bin/kalite --version) \ - && wine inno-compiler/ISCC.exe installer-source/KaliteSetupScript.iss \ - && cp /kalite/dist/* /kalitedist/ \ - && cp /kalite/ka-lite-installers/windows/KALiteSetup-$(/kalite/kalite_env/bin/kalite --version).exe /kalitedist/ +CMD cd /kalite && make dist pex && cp /kalite/dist/* /kalitedist/ diff --git a/Makefile b/Makefile index c276558876..0e455ccb61 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,9 @@ clean-test: rm -fr htmlcov/ clean-assets: - npm cache clean + # This is only relevant for npm<5 + # https://github.com/learningequality/ka-lite/issues/5519 + # npm cache clean rm -rf kalite/database/templates/ rm -rf .kalite_dist_tmp @@ -98,7 +100,7 @@ assets: bin/kalite manage retrievecontentpack empty en --foreground --template msgids: - export IGNORE_PATTERNS="*/kalite/static-libraries/*,*/LC_MESSAGES/*,*/kalite/packages/dist/*,*/kalite/packages/bundled/django/*,*/kalite/*bundle*.js,*/kalite/*/js/i18n/*.js" ;\ + export IGNORE_PATTERNS="*/kalite/static-libraries/*,*/LC_MESSAGES/*,*/kalite/packages/dist/*,*/kalite/packages/bundled/django/*,*/kalite/*/bundles/bundle*.js,*/kalite/*/js/i18n/*.js" ;\ cd kalite ;\ kalite manage makemessages -len --no-obsolete ;\ kalite manage makemessages -len --no-obsolete --domain=djangojs @@ -125,6 +127,10 @@ dist: clean docs assets python setup.py bdist_wheel --static --no-clean ls -l dist +pex: + pex -m kalite dist/ka_lite_static*.whl --disable-cache -o dist/ka-lite-static$$(git describe).pex + ls -l dist + install: clean python setup.py install diff --git a/README.rst b/README.rst index 5989be424a..224118a5cd 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ by `Learning EqualityKA Lite is community-driven and relies on help and experience which you can both share and receive in our community.
+