Merge remote-tracking branch 'upstream/master' into origo_argus_colos…

…sus_operational_status_feat

.env

@@ -123,6 +123,12 @@ APP_PRIVATE_KEY=this-is-not-so-secret-change-it
 TRUSTED_REVERSE_PROXIES=1
 # =====================================================================================
 
+## Telemetry
+# yes/no
+TELEMETRY_ENABLED=no
+# TELEMETRY_ENDPOINT=http://apm-server:8200
+TELEMETRY_ENDPOINT=http://collector:4318
+
 # joystream/node docker image tag
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.

.github/workflows/deploy-node-network.yml

@@ -86,6 +86,10 @@ jobs:
             EC2InstanceType=${{ steps.network_config.outputs.instanceType }},
             VolumeSize=${{ steps.network_config.outputs.volumeSize }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Prepare inventory for Ansible
         run: |
           VAL1="${{ steps.deploy_stack.outputs.Val1PublicIp }}"

.github/workflows/deploy-playground.yml

@@ -86,6 +86,10 @@ jobs:
           # Make sure ami image is available in the region specified in configure aws creds step
           parameter-overrides: 'KeyName=joystream-github-action-key-new,EC2InstanceType=t2.xlarge,EC2AMI=ami-06b4d9ba1f23a8da4'
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:

.github/workflows/joystream-apps-docker.yml

@@ -24,6 +24,14 @@ jobs:
           echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
           echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
 
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
@@ -74,14 +82,7 @@ jobs:
         run: |
           yarn build:packages
           cp docker-compose-no-bind-volumes.yml docker-compose.yml
-          ./tests/network-tests/run-test-node-docker.sh
-          sleep 15
-          docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
-          sleep 30
-          ./tests/network-tests/start-storage.sh
-          export REUSE_KEYS=true
-          export SKIP_STORAGE_AND_DISTRIBUTION=true
-          IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
+          ./tests/network-tests/run-tests.sh content-directory
 
       - name: Push new versions
         if: github.event_name == 'workflow_dispatch'
@@ -97,6 +98,6 @@ jobs:
         if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
         run: |
           docker push joystream/storage-node:latest
-          docker push joystream/distributor:latest
+          docker push joystream/distributor-node:latest
           docker push joystream/query-node:latest
 

.github/workflows/joystream-cli.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
@@ -32,7 +32,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/joystream-node-docker-publish.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum

.github/workflows/joystream-node-docker.yml

@@ -28,7 +28,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum
@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       # docker manifest inspect requires authentication
       - name: Login to DockerHub
@@ -118,6 +118,10 @@ jobs:
           parameter-overrides: 'KeyName=${{ env.KEY_NAME }},EC2AMI=${{ matrix.ec2ami }},EC2InstanceType=${{ matrix.ec2type }}'
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:

.github/workflows/joystream-node.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - uses: technote-space/get-diff-action@v3
         with:
           PREFIX_FILTER: |
@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - uses: technote-space/get-diff-action@v3
         with:
           PREFIX_FILTER: |

.github/workflows/joystream-types.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -30,7 +30,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/lint-typescript.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -24,7 +24,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/metadata-protobuf.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/query-node.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -27,7 +27,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/run-network-tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -28,7 +28,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -50,7 +50,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum
@@ -155,7 +155,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - name: Get artifacts
         uses: actions/download-artifact@v3
         with:
@@ -227,7 +227,7 @@ jobs:
       - name: setup node
         uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - name: Install packages and dependencies
         if: env.GIT_DIFF
         run: |

.github/workflows/storage-node.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -28,7 +28,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

CHANGELOG.md

@@ -0,0 +1,24 @@
+# Joystream Monorepo ChangeLog
+
+This is a top-level view of changes in the repo, linking to detailed changelogs of individual packages changed.
+If no packages or cargo crates are changed then the changes are usually "devops" related.
+
+### 2023-7-20 - Node.js update
+
+This is developer tools update release.
+
+#### Moving to node v18.6.0 from v14
+This is a potentially breaking change for your setup, since it requires a minimum GLIBC_2.28 on gnu/linux systems. If you use Ubuntu then you should be at least on the `20.04` release.
+
+To check your glibc version use this command: `ldd --version`
+
+If your version is older than v2.28 you should really consider updating your OS it is probably quite dated and vulnerable to security bugs.
+
+The minimum required node engine for all monorepo packages is updated to v14.18.0
+
+No npm packages have not been published with new engine version yet, so you are not impacted if your application depends on them, until the next package version release. To be ready do one of:
+
+1. Update to latest version of nodejs if possible, or
+1. If you have to stick with v14 update to the latest v14 release
+
+If you are using volta and your OS distro is fairly new you will not need to do anything, volta will automatically fetch and use required version. For workarounds see the install development tools section in the [README.md](./README.md)

README.md

@@ -9,30 +9,55 @@ functionality to support the [various roles](https://joystream.gitbook.io/testne
 
 ## Development
 
+For best results use GNU/Linux with minimum glibc version 2.28 for nodejs v18 to work.
+So Ubuntu 20.04 or newer.
+
+You can check your version of glibc with `ldd --version`
+
 The following tools are required for building, testing and contributing to this repo:
 
 - [Rust](https://www.rust-lang.org/tools/install) toolchain - _required_
-- [nodejs](https://nodejs.org/) v14.x - _required_
+- [nodejs](https://nodejs.org/) >= v14.18.x - _required_ (However volta will try to use v18.6)
 - [yarn classic](https://classic.yarnpkg.com/en/docs/install) package manager v1.22.x- _required_
-- [docker](https://www.docker.com/get-started) and docker-compose v.1.29 or higher - _required_
+- [docker](https://www.docker.com/get-started) and docker-compose v2.20.x or higher - _required_
 - [ansible](https://www.ansible.com/) - _optional_
 
 If you use VSCode as your code editor we recommend using the workspace [settings](devops/vscode/settings.json) for recommend eslint plugin to function properly.
 
-After cloning the repo run the following initialization scripts:
+After cloning the repo run the following to get started:
 
+### Install development tools
 ```sh
-# Install development tools
 ./setup.sh
+```
+
+### If you prefer your own node version manager
+Install development tools without Volta version manager.
 
-# build local npm packages
+```sh
+./setup.sh --no-volta
+```
+
+### For older operating systems which don't support node 18
+Modify the root `package.json` and change volta section to use node version 16.20.1 instead of 18.6.0
+```json
+"volta": {
+    "node": "16.20.1",
+    "yarn": "1.22.19"
+}
+```
+
+### Run local development network
+
+```sh
+# Build local npm packages
 yarn build:packages
 
-# Build joystream/node docker image
-yarn build:node:docker
+# Build joystream/node docker testing image
+RUNTIME_PROFILE=TESTING yarn build:node:docker
 
-# start a local development network
-yarn start
+# Start a local development network
+RUNTIME_PROFILE=TESTING yarn start
 ```
 
 ## Software
@@ -72,7 +97,15 @@ WASM_BUILD_TOOLCHAIN=nightly-2022-05-11 cargo build --release
 
 Learn more about [joystream-node](bin/node/README.md).
 
-A step by step guide to setup a full node and validator on the Joystream testnet, can be found [here](https://joystream.gitbook.io/testnet-workspace/system/validation#validator).
+A step by step guide to setup a full node and validator on the Joystream main network, can be found [here](https://handbook.joystream.org/system/validation).
+
+### Pre-built joystream-node binaries
+Look under the 'Assets' section:
+
+- Ephesus release [v8.3.0](https://github.com/Joystream/joystream/releases/tag/v12.2001.0)
+
+### Mainnet chainspec file
+- [joy-mainnet.json](https://github.com/Joystream/joystream/releases/download/v12.1000.0/joy-mainnet.json)
 
 ### Integration tests