diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 45799bd73..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,67 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '21 0 * * 4' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] - # Learn more: - # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be251ae20..20ada36d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,10 @@ name: ci -on: pull_request +on: + pull_request: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + jobs: lint: @@ -10,18 +15,41 @@ jobs: - uses: actions/setup-python@v1 with: python-version: '3.7' - - run: make install tools + - name: extract pip cache + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} + restore-keys: ${{ runner.os }}-pip- + - run: pip install --user --upgrade pip wheel + - run: pip install -e .[torrent,lint] - run: make lint tests-unit: name: "tests / unit" - runs-on: ubuntu-latest + strategy: + matrix: + os: + - ubuntu-latest + #- macos-latest + #- windows-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: python-version: '3.7' - - run: make install tools + - name: set pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + - name: extract pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} + restore-keys: ${{ runner.os }}-pip- + - run: pip install --user --upgrade pip wheel + - run: pip install -e .[torrent,test] - env: HOME: /tmp run: make test-unit-coverage @@ -38,15 +66,14 @@ jobs: steps: - name: Configure sysctl limits run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 - + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 - name: Runs Elasticsearch uses: elastic/elastic-github-actions/elasticsearch@master with: - stack-version: 7.6.0 + stack-version: 7.6.0 - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: @@ -55,16 +82,22 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends ffmpeg - - run: pip install tox-travis + - name: extract pip cache + uses: actions/cache@v2 + with: + path: ./.tox + key: tox-integration-${{ matrix.test }}-${{ hashFiles('setup.py') }} + restore-keys: txo-integration-${{ matrix.test }}- + - run: pip install tox - run: tox -e ${{ matrix.test }} build: needs: ["lint", "tests-unit", "tests-integration"] - name: "build" + name: "build / binary" strategy: matrix: os: - - ubuntu-latest + - ubuntu-16.04 - macos-latest - windows-latest runs-on: ${{ matrix.os }} @@ -73,22 +106,80 @@ jobs: - uses: actions/setup-python@v1 with: python-version: '3.7' - - name: Setup - run: | - pip install pyinstaller - pip install -e . - # https://stackoverflow.com/a/61693590 - # https://github.com/pypa/setuptools/issues/1963 - pip install --upgrade 'setuptools<45.0.0' - - if: startsWith(matrix.os, 'windows') == false + - id: os-name + uses: ASzc/change-string-case-action@v1 + with: + string: ${{ runner.os }} + - name: set pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + - name: extract pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} + restore-keys: ${{ runner.os }}-pip- + - run: pip install pyinstaller==4.4 + - run: pip install -e . + - if: startsWith(github.ref, 'refs/tags/v') + run: python docker/set_build.py + - if: startsWith(runner.os, 'linux') || startsWith(runner.os, 'mac') name: Build & Run (Unix) run: | pyinstaller --onefile --name lbrynet lbry/extras/cli.py - chmod +x dist/lbrynet dist/lbrynet --version - - if: startsWith(matrix.os, 'windows') + - if: startsWith(runner.os, 'windows') name: Build & Run (Windows) run: | - pip install pywin32 + pip install pywin32==301 pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico --onefile --name lbrynet lbry/extras/cli.py dist/lbrynet.exe --version + - uses: actions/upload-artifact@v2 + with: + name: lbrynet-${{ steps.os-name.outputs.lowercase }} + path: dist/ + + docker: + name: "build / docker" + if: startsWith(github.ref, 'refs/tags/v') + needs: ["build"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/download-artifact@v2 + with: + name: lbrynet-linux + - run: | + chmod +x lbrynet + mv lbrynet docker + - name: build and push docker image + uses: docker/build-push-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: lbry/lbrynet + path: docker + dockerfile: docker/Dockerfile.wallet_server + tag_with_ref: true + tag_with_sha: true + add_git_labels: true + + release: + name: "release" + if: startsWith(github.ref, 'refs/tags/v') + needs: ["build"] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v2 + - name: zip binaries + run: | + chmod +x lbrynet-macos/lbrynet + chmod +x lbrynet-linux/lbrynet + zip --junk-paths lbrynet-mac.zip lbrynet-macos/lbrynet + zip --junk-paths lbrynet-linux.zip lbrynet-linux/lbrynet + zip --junk-paths lbrynet-windows.zip lbrynet-windows/lbrynet.exe + ls -lh + - run: gh release upload ${GITHUB_REF#refs/tags/} lbrynet-mac.zip lbrynet-linux.zip lbrynet-windows.zip + env: + GITHUB_TOKEN: ${{ github.token }} + diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml deleted file mode 100644 index bc77715f6..000000000 --- a/.github/workflows/ossar-analysis.yml +++ /dev/null @@ -1,44 +0,0 @@ -# This workflow integrates a collection of open source static analysis tools -# with GitHub code scanning. For documentation, or to provide feedback, visit -# https://github.com/github/ossar-action -name: OSSAR - -on: - push: - branches: [ master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] - schedule: - - cron: '28 17 * * 1' - -jobs: - OSSAR-Scan: - # OSSAR runs on windows-latest. - # ubuntu-latest and macos-latest support coming soon - runs-on: windows-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Ensure a compatible version of dotnet is installed. - # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. - # A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action. - # GitHub hosted runners already have a compatible version of dotnet installed and this step may be skipped. - # For self-hosted runners, ensure dotnet version 3.1.201 or later is installed by including this action: - # - name: Install .NET - # uses: actions/setup-dotnet@v1 - # with: - # dotnet-version: '3.1.x' - - # Run open source static analysis tools - - name: Run OSSAR - uses: github/ossar-action@v1 - id: ossar - - # Upload results to the Security tab - - name: Upload OSSAR results - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: ${{ steps.ossar.outputs.sarifFile }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..f94c3652e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,17 @@ +name: slack +on: + release: + types: [published] + +jobs: + release: + name: "slack notification" + runs-on: ubuntu-latest + steps: + - name: send message + uses: slackapi/slack-github-action@v1.14.0 + with: + payload: '{"type": "mrkdwn", "text":"Hello, *World*!"}' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE_BOT_WEBHOOK }} + diff --git a/Makefile b/Makefile index 0a2bf7c15..b3e7df236 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ .PHONY: install tools lint test test-unit test-unit-coverage test-integration idea install: - pip install lbry-libtorrent pip install -e . tools: diff --git a/setup.py b/setup.py index 0bf48ac02..7646d1a3e 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,13 @@ setup( 'pbkdf2==1.3', 'attrs==18.2.0', 'pylru==1.1.0', - 'elasticsearch==7.10.1' + 'elasticsearch==7.10.1', ] + PLYVEL, + extras_require={ + 'torrent': ['lbry-libtorrent'], + 'lint': ['mypy==0.701', 'pylint==2.4.4'], + 'test': ['coverage'], + }, classifiers=[ 'Framework :: AsyncIO', 'Intended Audience :: Developers', diff --git a/tox.ini b/tox.ini index 95c3afa5b..0b8b7ff90 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,14 @@ [testenv] deps = coverage - -extras = test +extras = + test + torrent changedir = {toxinidir}/tests setenv = HOME=/tmp ELASTIC_HOST={env:ELASTIC_HOST:localhost} commands = - pip install https://github.com/rogerbinns/apsw/releases/download/3.30.1-r1/apsw-3.30.1-r1.zip \ - --global-option=fetch \ - --global-option=--version --global-option=3.30.1 --global-option=--all \ - --global-option=build --global-option=--enable --global-option=fts5 - pip install lbry-libtorrent orchstr8 download blockchain: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.blockchain {posargs} datanetwork: coverage run -p --source={envsitepackagesdir}/lbry -m unittest discover -vv integration.datanetwork {posargs}