Merge branch 'feat/semantic-release-and-docs' into 'develop'

Add semantic release features

See merge request sibmip/gateway!30

.gitlab-ci.yml

-image: docker:20
-services:
-  - docker:20-dind
-
 stages:
   - build
   - release
-  - deploy
 
-variables:
-  CONTAINER_BASE_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+services:
+  - docker:dind
 
 before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build:
+  image: docker:dind
   stage: build
-  script:
-    - docker build -t $CONTAINER_BASE_IMAGE ./api
-    - docker push $CONTAINER_BASE_IMAGE
   only:
+    refs:
     - main
+    - rc
+    - beta
     - develop
-    - /^release-.*$/
+    # This matches maintenance branches
+    - /^(([0-9]+)\.)?([0-9]+)\.x/
+    # This matches pre-releases
+    - /^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+  script:
+    - docker build -t $CI_REGISTRY/$CI_PROJECT_PATH:pipeline-$CI_PIPELINE_ID ./api
+    - docker push $CI_REGISTRY/$CI_PROJECT_PATH:pipeline-$CI_PIPELINE_ID
+
+release:
+  image: docker:dind
+  stage: release
+  only:
+    refs:
+    - main
+    - rc
+    - beta
+    # This matches maintenance branches
+    - /^(([0-9]+)\.)?([0-9]+)\.x/
+    # This matches pre-releases
+    - /^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+  script:
+    - apk add --no-cache nodejs npm py3-pip python2 git
+    - docker pull $CI_REGISTRY/$CI_PROJECT_PATH:pipeline-$CI_PIPELINE_ID
+    - docker tag $CI_REGISTRY/$CI_PROJECT_PATH:pipeline-$CI_PIPELINE_ID gateway-semantic:latest
+    - cd ./api
+    - npm install @semantic-release/gitlab @semantic-release/changelog @eclass/semantic-release-docker
+    - npx semantic-release
\ No newline at end of file

README.md

+# 🏠 Home
+
+[![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release)
+[![pipeline status](https://gitlab.com/sibmip/gateway/badges/main/pipeline.svg)](https://gitlab.com/sibmip/gateway/-/commits/main)
+
+## Introduction
+
+The MIP Gateway is a middleware layer between the [MIP Frontend](https://github.com/HBPMedical/portal-frontend) and a federate analytic engine (Exareme, Datashield, FATE, MedCo, etc...).
+
+## Contact
+* [Manuel Spuhler](https://github.com/nicedexter) (<manuel.spuhler@chuv.ch>)
+* [Steve Mendes Reis](https://github.com/M4n0x) (<steve.mendes-reis@chuv.ch>)
+
+## License
+Licensed under the GNU Affero General Public License, Version 3.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at [https://www.gnu.org/licenses/agpl-3.0.html](https://www.gnu.org/licenses/agpl-3.0.html)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Technical documentation 
+
+Technical documentation can be found at [https://mip-front.gitbook.io/mip-gateway-doc/](https://mip-front.gitbook.io/mip-gateway-doc/)
+
+

SUMMARY.md

+# Table of contents
+
+* [🏠 Home](README.md)
+
+## For Developers
+
+* [Get Started](docs/for-developers/get-started/README.md)
+  * [Introduction](docs/get-started/Introduction.md)
+  * [Setup development environment](docs/get-started/Setup-development-environment.md)
+* [Connector](docs/for-developers/connector/README.md)
+  * [Parsing response with JSONdata](docs/gateway/Connector/Parsing-response-with-JSONata.md)
+* [Frontend](docs/for-developers/frontend/README.md)
+  * [Update GraphQL Queries](docs/frontend/Update-queries-GrahpQL-in-the-frontend.md)

api/package.json

@@ -39,9 +39,13 @@
     "rxjs": "^7.2.0"
   },
   "devDependencies": {
+    "@eclass/semantic-release-docker": "^2.0.0",
     "@nestjs/cli": "^8.0.0",
     "@nestjs/schematics": "^8.0.0",
     "@nestjs/testing": "^8.0.0",
+    "@semantic-release/changelog": "^6.0.1",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/gitlab": "^7.0.4",
     "@types/express": "^4.17.13",
     "@types/jest": "^27.0.1",
     "@types/node": "^16.0.0",
@@ -77,5 +81,54 @@
     ],
     "coverageDirectory": "../coverage",
     "testEnvironment": "node"
+  },
+  "release": {
+    "plugins": [
+      "@semantic-release/commit-analyzer",
+      "@semantic-release/release-notes-generator",
+      "@semantic-release/changelog",
+      "@semantic-release/gitlab",
+      "@semantic-release/npm",
+      [
+        "@semantic-release/git",
+        {
+          "assets": [
+            "./dist",
+            "CHANGELOG.md",
+            "package.json",
+            "package-lock.json"
+          ]
+        }
+      ],
+      [
+        "@eclass/semantic-release-docker",
+        {
+          "baseImageName": "gateway",
+          "registries": [
+            {
+              "url": "docker.io",
+              "imageName": "docker.io/m4n0x/gateway",
+              "user": "DOCKER_REGISTRY_USER",
+              "password": "DOCKER_REGISTRY_PASSWORD",
+              "skipTags": [
+                "latest"
+              ]
+            }
+          ]
+        }
+      ]
+    ],
+    "branches": [
+      "main",
+      "+([0-9])?(.{+([0-9]),x}).x",
+      {
+        "name": "rc",
+        "prerelease": true
+      },
+      {
+        "name": "beta",
+        "prerelease": true
+      }
+    ]
   }
 }

docs/for-developers/connector/README.md

+# Connector
+

docs/for-developers/frontend/README.md

+# Frontend
+

docs/for-developers/get-started/README.md

+# Get Started
+

docs/frontend/Update-queries-GrahpQL-in-the-frontend.md

+# Update GraphQL Queries
+
+## New method
+
+The context of the this page is related to the portal-frontend not the gateway directly.
+
+In order to update or create new types and operations, you can proceed with the following command :
+
+```bash
+yarn codegen
+```
+
+This command will generate all the operations and types for you. You should place the graphql's operations in the file queries.ts under the folder `src/components/API/GraphQL`. Types will be all generated in one file under the name `types.generated.ts`, operations will be in `queries.generated.tsx` file and all fragments under the file `fragments.generated.tsx`.
+
+***
+
+## Previous method
+
+This method is no more needed as the front-portal has been updated and all the commands can be made on the front directly.
+
+### Context
+
+_(This guide mainly follow the procedure describe here : https://blog.logrocket.com/build-graphql-react-app-typescript/)_
+
+This guide is made to generate types and hooks to query/mutate for the frontend based on the GraphQL API. This tool is used to avoid to do it manually.
+
+The generation could be made directly from the frontend project but due to the old dependencies in the current frontend project it does not work.
+
+The guide, that will be describe here, is a workaround to generate the types/hooks outside of the front project.
+
+### Procedure
+
+First of all make sur that you have npm and yarn installed on your system.
+
+The first to do is to create a new react typescript project
+
+#### Setup dependencies
+
+Create an empty folder and run after these commands from the newly created folder :
+
+`yarn add @apollo/client graphql`
+
+`yarn add -D @graphql-codegen/cli`
+
+It will setup all the dependencies needed to generated the types and hooks.
+
+#### Declare GrahpQL queries
+
+To generate the types and hooks, you need to provide the queries that you'll be using, so you need to put a file named `queries.ts`, an example is provided :
+
+```ts
+import { gql } from '@apollo/client';
+
+export const QUERY_DOMAINS = gql`
+  query listDomains {
+    domains {
+      id
+    }
+  }
+`;
+
+export const QUERY2 = gql`
+  query listVariables {
+    domains {
+      variables {
+        id
+      }
+    }
+  }
+`;
+
+...
+```
+
+#### Init codegen configuration
+
+To init codegen configuration, you can enter the following command
+
+`npx graphql-codegen init`
+
+After that you will need to provide some information :
+
+* What type of application are you building ?
+  * Choose `Application built with React`
+* Where is your schema ?
+  * Give the url of your graphql's endpoint (default : http://127.0.0.1:8081/graphql)
+* Where are your operations and fragments ?
+  * `./queries.ts`
+* Pick plugins
+  * Let the default 3 plugins : TS, TS Operations, TS React Apollo
+* Where to write the output
+  * `src/generated/graphql.tsx` (default)
+* Do you want to generated an introspection file ?
+  * no
+* How to name the config file ?
+  * `codegen.yml`
+* What script in package.json should run the codegen
+  * `codegen`
+
+After this process you will need to run `yarn install` in order to install the new dependencies that have been added to the package.json.
+
+#### Generate and integrate
+
+Everything is now configured, you just need to run `yarn codegen` it will generated all you need in ./src/generated/graphql.tsx. You can copy the content of the generated replace the previous one if there is one or just create a new file under /src/generated/graphql.tsx
+
+If an error occurs telling you `Unable to find template plugin matching typescript-operations` you should try to run this command `npm i -D @graphql-codegen/typescript change-case` (see issue in the link section) and retry the previous command.
+
+**Update queries.ts**
+
+If you needed to regenerate the GrahpQL types and hooks you can keep your folder that is already setup and just change the content of the queries.ts, re-run `yarn codegen` and that's it.
+
+### Links
+
+* [GraphQL code generator](https://www.graphql-code-generator.com)
+  * [GitHub](https://github.com/dotansimha/graphql-code-generator)
+* [Issue: unable to find typescript-operations](https://github.com/dotansimha/graphql-code-generator/issues/2043)

docs/gateway/Connector/Add-a-connector.md

+# Introduction
+
+Todo
\ No newline at end of file

docs/gateway/Connector/Parsing-response-with-JSONata.md

+# Parsing response with JSONdata
+
+This page is dedicated to talk about the transformation of JSON data to fit the Gateway needs.
+
+Most of the work done by the gateway is about transformation. The data receive by the engine is transformed in a way that could be consume by the front-end. There is a lot of work when adapting an old engine, most of the work can be achieve in JS/TS but it can be really time consuming and most of the time the code for data transformation is not really readable.
+
+For this purpose we suggest to use a library to transform the data (JSON in that case) to fit the Gateway's models.
+
+## JSONata
+
+In order to simplify the transformation of JSON data, we choose the library [JSONata](https://jsonata.org) to do this work. JSONata is :
+
+* Lightweight query and transformation language for JSON data
+* Inspired by the location path semantics of XPath 3.1
+* Sophisticated query expressions with minimal syntax
+* Built in operators and functions for manipulating and combining data
+* Create user-defined functions
+* Format query results into any JSON output structure
+
+It makes the transformation really easy to do, more readable and thus maintainable.
+
+### Example
+
+File _transformation.ts_
+
+```ts
+import * as jsonata from 'jsonata';
+
+export const expression = jsonata(`
+  $sum(Account.Order.Product.(Price * Quantity))
+`);
+```
+
+File _converter.ts_
+
+```ts
+import { expression } from './transformations';
+
+const data = `
+{
+  "Account": {
+    "Account Name": "Firefly",
+    "Order": [
+      {
+        "OrderID": "order103",
+        "Product": [
+          {
+            "Product Name": "Bowler Hat",
+            "ProductID": 858383,
+            "SKU": "0406654608",
+            "Description": {
+              "Colour": "Purple",
+              "Width": 300,
+              "Height": 200,
+              "Depth": 210,
+              "Weight": 0.75
+            },
+            "Price": 34.45,
+            "Quantity": 2
+          },
+          {
+            "Product Name": "Trilby hat",
+            "ProductID": 858236,
+            "SKU": "0406634348",
+            "Description": {
+              "Colour": "Orange",
+              "Width": 300,
+              "Height": 200,
+              "Depth": 210,
+              "Weight": 0.6
+            },
+            "Price": 21.67,
+            "Quantity": 1
+          }
+        ]
+      }
+    ]
+  }
+}
+`;
+
+const value: SOME_TYPE = expression.evaluate(data);
+
+console.log(value);
+```
+
+### Conception / Debugging
+
+You can try transformations and debug jsonata directly with the online tool : [https://try.jsonata.org](https://try.jsonata.org).

docs/get-started/Introduction.md

+---
+description: Introduction for developers
+---
+
+# Introduction
+
+The MIP is mainly composed by 3 components
+
+![](../.gitbook/assets/components.drawio.png)
+
+* **Frontend** : user interface (React.js)
+* **Gateway** : middleware used to abstract calls from an engine (Nest.js and GraphQL)
+* **Engine** : an API to a concrete engine (Datashield, Exareme, etc...)
+
+## Overview
+
+The Frontend and the Gateway are closely related and their communication are abstracted from the engine. This abstraction is performed by the Gateway.
+
+![](../.gitbook/assets/overview.png)
+
+The Gateway is in charge of communications with various engines, it could make simple calls and pass it to the Frontend or make any transformations needed in order to fit the need in the Frontend.
+
+### GraphQL
+
+GraphQL is used to provide some useful features :
+
+* Shared types/interfaces between Gateway and Frontend
+  * With this feature we don't need to redefine every type. Every type/interface are extracted directly from GraphQL endpoint.
+* Fully documented endpoint
+  * This feature allows to keep track of possibilities offer by the gateway, as this is directly extracted from the code, the description of the types/interfaces and the different calls (query, mutation, subscription) are always up to date.
+* Better flexibility in the Frontend
+  * We can actually select information needed on every request, that means better separation of concern and isolation of React's components
+  * Along with the use of Apollo Client, GraphQL provide a new way to manage State Management
+
+### Connectors
+
+In order to be able to communicate with an engine, the gateway need connectors. A connector is a concrete implementation of the interface `IEngineService`.
+
+```typescript
+export default class DatashieldService implements IEngineService {
+  constructor(
+    private readonly options: IEngineOptions,
+    private readonly httpService: HttpService,
+  ) {}
+
+  ...
+}
+```
+
+The code above is an example of a connector.
+
+### Visualizations
+
+![](../.gitbook/assets/visualizations.png)
+
+With the Frontend we will introduce a new way to deal with visualizations. Previously the visualizations were completely manage by the engine. As a part of abstraction from a specific engine we want to be able to delegate this task to the visualization components.
+
+The main idea is to build various generalist visualization components (BarChart, HeatMap, DataTable, etc...) that every analytic engine can either feed directly or be adapted by the Gateway in order to fit the Frontend needs.

docs/get-started/Setup-development-environment.md

+---
+description: Developer guide to setup a development environment
+---
+
+# Setup development environment
+
+This guide is used to setup a full development environment for the MIP Front dev.
+
+## Introduction
+
+As we have seen in the previous chapter we need three component in order to setup the MIP:
+
+* Engine
+* Frontend
+* Gateway
+
+In this guide we will see how to setup the last two elements.
+
+### Prerequisites
+
+Make sure to have
+
+* [Node.js](https://nodejs.org)
+* [NPM](https://npmjs.com)
+* [Yarn](https://yarnpkg.com)
+
+installed in your computer.
+
+## Setup Gateway
+
+First of all you should clone the repository 'gateway' either from the [Gitlab](https://gitlab.com/sibmip/gateway) or from the [GitHub](https://github.com/HBPMedical/gateway).
+
+Once the pull is completed, you can make the following commands
+
+```bash
+cd gateway/api
+
+git checkout develop
+
+npm install
+```
+
+After these steps you should be able to start the gateway in dev mode with the following command
+
+```bash
+npm run start:dev
+```
+
+### env.defaults
+
+There is an environment file that allows some configuration for the gateway.
+
+```yaml
+ENGINE_TYPE=local
+ENGINE_BASE_URL=http://127.0.0.1:8080/services/
+GATEWAY_PORT=8081
+```
+
+* ENGINE\_TYPE
+  * Allows you to choose which connector you want to load, `local` is used for development purpose and is not intended to be used in production.
+* ENGINE\_BASE\_URL
+  * Indicate the endpoint for the engine, this parameter can be retrieve in the connector side. If you are using `local` connector this parameter is not useful.
+* GATEWAY\_PORT
+  * Indicate the port for the Gateway.
+
+These parameters can be overwrite by either :
+
+* setting a variable in `.env` file (you can create it if it does not exist) along with the file .env.defaults in the root folder
+* or setting an environment variable on your system
+
+### GraphQL Playground
+
+Once you have started the Gateway, you can play with the GraphQL playground that is automatically integrated within the gateway, follow this link : [http://127.0.0.1:8081/graphql](http://127.0.0.1:8081/graphql). You should be able to see something like this :
+
+![GraphQL Playground](../.gitbook/assets/image.png)
+
+This environment is a tool provided by GraphQL to play with queries, mutations, etc...
+
+It allows you to see schema defined in the backend and endpoints that you can calls.
+
+## Setup Frontend
+
+First of all you should clone the repository 'portal-frontend' either from the [Gitlab](https://gitlab.com/sibmip/portal-frontend) or the [GitHub](https://github.com/HBPMedical/portal-frontend).
+
+Once the pull is completed, you can make the following commands
+
+```bash
+cd portal-frontend
+
+git checkout dev
+
+yarn install
+```
+
+### Environment file
+
+Make sure that the .env or .env.development contains the following information
+
+```yaml
+REACT_APP_BACKEND_URL=http://127.0.0.1:8081
+REACT_APP_GATEWAY_URL=$REACT_APP_BACKEND_URL/graphql
+```
+
+* REACT\_APP\_BACKEND\_URL
+  * Is used to consume REST API call (migration to GraphQL is in progress)
+* REACT\_APP\_GATEWAY\_URL
+  * Is used to connect with GraphQL Gateway
+
+### Start frontend
+
+Depending the branch you are working on, you can start the Frontend with
+
+```bash
+yarn start
+```
+
+or
+
+```bash
+yarn watch
+```
+
+## Launch order
+
+Each service could be launch at any time without respecting any specific order, however as the Frontend (react app) open automatically a browser tab for the local development, it is recommended to follow this launch order
+
+`Engine -> Gateway -> Frontend`
+
+If you are using `local` type in the Gateway you don't need to start the engine.