Skip to content

Instantly share code, notes, and snippets.

@alexey-milovidov

alexey-milovidov/docs.py

Created February 27, 2021 14:50
Show Gist options
  • Save alexey-milovidov/4a3365fbc95d5481dcefe75a90fa1182 to your computer and use it in GitHub Desktop.
Save alexey-milovidov/4a3365fbc95d5481dcefe75a90fa1182 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
docs.py
# -*- coding: utf-8 -*-
import logging
import sandbox.sdk2.helpers
import subprocess
import os
from sandbox import sdk2
import sandbox.common.types.resource as ctr
import sandbox.common.types.client as ctc
from sandbox.projects.clickhouse.BaseOnCommitTask.base import NeedToRunDescription
from sandbox.projects.clickhouse.BaseOnCommitTask.base_text_test_task import BaseTextTestTask
from sandbox.projects.clickhouse.resources import CLICKHOUSE_DOCS_LXC_CONTAINER
from sandbox.projects.clickhouse.util.task_helper import has_changes_in_documentation
class ClickhouseDocsCheck(BaseTextTestTask):
class Requirements(BaseTextTestTask.Requirements):
privileged = True
client_tags = ctc.Tag.GENERIC
class Parameters(BaseTextTestTask.Parameters):
_container = sdk2.parameters.Container(
"Environment container resource",
resource_type=CLICKHOUSE_DOCS_LXC_CONTAINER,
)
def on_create(self):
self.Parameters._container = sdk2.Resource.find(
CLICKHOUSE_DOCS_LXC_CONTAINER,
state=ctr.State.READY,
attrs=dict(released="stable")
).order(-CLICKHOUSE_DOCS_LXC_CONTAINER.id).first().id
@staticmethod
def need_to_run(pr_info):
if 'pr-backport' in pr_info.labels or 'release' in pr_info.labels:
return NeedToRunDescription(False, 'Not ran for backport or release PRs', True)
if not has_changes_in_documentation(pr_info):
return NeedToRunDescription(False, "No diff in .md, .html, etc...", True)
return NeedToRunDescription(True)
@staticmethod
def get_context_name():
return "Docs check"
def install_newer_python(self):
with sandbox.sdk2.helpers.ProcessLog(self, logger="insall_python") as pl:
cmd = "add-apt-repository ppa:deadsnakes/ppa -y && "\
" apt-get update && apt-get -o DPkg::Options::=\"--force-confold\" install python3.7 python3.7-venv --yes --force-yes && "\
" update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && python3 -m ensurepip --upgrade"
retcode = subprocess.Popen(cmd, shell=True, stderr=pl.stdout, stdout=pl.stdout).wait()
if retcode != 0:
raise Exception("Cannot prepare new python")
def check(self, repo_path, commit, repo, pull_request):
logging.info("Docs check started")
output_name = "docs_output.txt"
lines = []
self.install_newer_python()
with sandbox.sdk2.helpers.ProcessLog(self, logger="docs_output") as pl:
retcode = subprocess.Popen(
"/bin/bash -c \"set -o pipefail && cd {}/docs/tools && python3 -m pip install --ignore-installed --upgrade setuptools pip virtualenv \
&& mkdir venv && virtualenv -p $(which python3) venv && source venv/bin/activate \
&& python3 -m pip install --ignore-installed -r requirements.txt && ./build.py --skip-git-log 2>&1 | tee {}\"".format(repo_path, output_name),
shell=True, stderr=pl.stdout, stdout=pl.stdout).wait()
if retcode == 0:
logging.info("Docs check finished")
else:
logging.info("Check failed")
lines.append(("Return code is not 0", "FAIL"))
log_path = str(pl.stdout.path)
output_path = "{}/docs/tools/{}".format(repo_path, output_name)
if os.path.exists(output_path):
logging.info("Output exists %s", output_path)
with open(output_path, 'r') as check_file:
for line in check_file:
if "ERROR" in line:
lines.append((line.split(':')[-1], "FAIL"))
else:
output_path = log_path
logging.info("Output %s doesn't exists", output_path)
return lines, output_path
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment