diff --git a/.env.prod.example b/.env.prod.example index 2132cbe70c62e853334ccdf3534f739a474ebc8a..6901cb63d7f1011463b08a393889079d1c4d41c8 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -1,15 +1,19 @@ # example .env file for docker-compose.prod.yml +# database configuration DB_USER=postgres DB_PASSWORD=postgrespassword DB_DATABASE=postgres + +# hasura configuration HASURA_LISTEN_PORT=8888 HASURA_GRAPHQL_ADMIN_SECRET=hasura_admin_secret -KUBO_GATEWAY_PORT=8080 -KUBO_DOMAIN=datapod.coinduf.eu -KUBO_GATEWAY_DOMAIN=gateway.datapod.coinduf.eu -KUBO_GATEWAY_SUBDOMAIN=pagu.re -SUBMIT_GATEWAY_LISTEN_PORT=3000 + +# kubo configuration +KUBO_DOMAIN=datapod.coinduf.eu # domain (used in p2p peering) +KUBO_GATEWAY_PORT=8080 # listen port of ipfs http gateway +KUBO_GATEWAY_DOMAIN=gateway.datapod.coinduf.eu # domain for kubo reverse proxy ipfs http gateway reverse proxy +KUBO_GATEWAY_SUBDOMAIN=pagu.re # domain for subdomain gateway (provides origin isolation but requires wildcard) # configure the node boot -DATAPOD_BOOT=bafyreih4jspnqnsd4o3sdqv7c765uyylhtlh5majjw6aq6clilkq7tmqey \ No newline at end of file +DATAPOD_BOOT=/ipfs/bafyreih4jspnqnsd4o3sdqv7c765uyylhtlh5majjw6aq6clilkq7tmqey \ No newline at end of file diff --git a/README.md b/README.md index 320b02a91a72816365f11a7bc335a17ed7860a1d..913a2b44d86cd394ca7ff7b698b5efc986e67aea 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This will pull preconfigured images for postgres/hasura, kubo and datapod. This - start collecting from default IPNS - index to database -You can then do a simple proxy_pass to `HASURA_LISTEN_PORT` and `KUBO_GATEWAY_PORT`. +You can then do a simple proxy_pass to `HASURA_LISTEN_PORT` and `KUBO_GATEWAY_PORT`. Read more on [install page](./doc/install.md). ## Dev diff --git a/doc/install.md b/doc/install.md new file mode 100644 index 0000000000000000000000000000000000000000..42fa8c204829f51b3a018397243dadfa3307b599 --- /dev/null +++ b/doc/install.md @@ -0,0 +1,34 @@ +# Install + +To install a datapod, use the [`docker-compose.prod.yml`](../docker-compose.prod.yml) and [`.env.prod.example`](../.env.prod.example) files: + +```sh +# edit env file +vim .env +# start services +docker compose up -d +# follow only datapod logs +docker compose logs -f datapod +``` + +The logs should give you: + +``` +# TODO +``` + +## Setup reverse proxy + +Depending on what you want to expose publicly, use the following nginx config files examples. + +- to expose graphql endpoint, see [`hasura.nginx.conf`](./nginx/hasura.nginx.conf) +- to expose ipfs http gateway, see [`ipfs-gateway.nginx.conf`](./nginx/ipfs-gateway.nginx.conf) +- to expose http gateway with subdomain, see [`subdomain-ipfs-gateway.nginx.conf`](./nginx/subdomain-ipfs-gateway.nginx.conf) (requires wildcard certificates) +- to expose rpc api, see [`ipfs-rpc.nginx.conf`](./nginx/ipfs-rpc.nginx.conf) (should not be done on main IPFS node) + +A bit of explanation about what they are: + +**graphql endpoint** Endpoint used for search capabilities. +**ipfs http gateway** A way to get IPFS content (image for example) through http (without embeding a node). +**subdomain gateway** Same as gateway, but provides origin isolation and root domain, for example for websites. +**rpc api** A way to "lend" an IPFS node to clients who do not embed one through a subset of the rpc API. This should be done on a different node than the one used by the datapod. It should also come with a rate limiter or some kind of protection mecanism. \ No newline at end of file diff --git a/doc/nginx/hasura.nginx.conf b/doc/nginx/hasura.nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..91f4f2521999fbd30e3055b8f63b833e87cf0168 --- /dev/null +++ b/doc/nginx/hasura.nginx.conf @@ -0,0 +1,36 @@ +# https graphql + wss subscriptions +# for HASURA_DOMAIN (replace by your domain name) +# on HASURA_LISTEN_PORT (as choosen in environment file) + +# http redirection +server { + listen 80 ; + listen [::]:80 ; + server_name <HASURA_DOMAIN>; + return 301 https://$host$request_uri; +} + +# listen on 443 +server { + listen 443 ssl; + listen [::]:443 ssl; + + # for example with letsencrypt + ssl_certificate /etc/letsencrypt/live/<HASURA_DOMAIN>/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/<HASURA_DOMAIN>/privkey.pem; + + server_name <HASURA_DOMAIN>; + + access_log /var/log/nginx/<HASURA_DOMAIN_access>.log; + error_log /var/log/nginx/<HASURA_DOMAIN_error>.log; + + location / { + add_header X-Robots-Tag "noindex"; + proxy_pass http://localhost:HASURA_LISTEN_PORT; + + proxy_set_header Host $host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/doc/nginx/ipfs-gateway.nginx.conf b/doc/nginx/ipfs-gateway.nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..616a77563e2636535ae13324d10a9a2e5788fc8b --- /dev/null +++ b/doc/nginx/ipfs-gateway.nginx.conf @@ -0,0 +1,31 @@ +# replace the following +# KUBO_GATEWAY_DOMAIN = domain of your ipfs http gateway +# KUBO_GATEWAY_PORT = port on which it is listening + +server { + listen 80 ; + listen [::]:80 ; + + server_name KUBO_GATEWAY_DOMAIN; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/letsencrypt/live/KUBO_GATEWAY_DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/KUBO_GATEWAY_DOMAIN/privkey.pem; + + server_name KUBO_GATEWAY_DOMAIN; + + access_log /var/log/nginx/KUBO_GATEWAY_DOMAIN_access.log; + error_log /var/log/nginx/KUBO_GATEWAY_DOMAIN_error.log; + + # main location is gateway + location / { + add_header X-Robots-Tag "noindex"; + proxy_pass http://localhost:KUBO_GATEWAY_PORT; + proxy_set_header Host $host; + } +} diff --git a/doc/nginx/ipfs-rpc.nginx.conf b/doc/nginx/ipfs-rpc.nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..88b93a5ee93205b476d5b4199342cf3a6d0abc2f --- /dev/null +++ b/doc/nginx/ipfs-rpc.nginx.conf @@ -0,0 +1,41 @@ +# replace the following: +# RPC_KUBO_DOMAIN = the name under which expose your RPC API +# RPC_KUBO_PORT = the rpc api port (ususally 5001) + +server { + listen 80 ; + listen [::]:80 ; + + server_name RPC_KUBO_DOMAIN; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/letsencrypt/live/RPC_KUBO_DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/RPC_KUBO_DOMAIN/privkey.pem; + + server_name RPC_KUBO_DOMAIN; + + access_log /var/log/nginx/RPC_KUBO_DOMAIN_access.log; + error_log /var/log/nginx/RPC_KUBO_DOMAIN_error.log; + + # main location + location / { + return 400; + } + location /api/v0/dag/ { + proxy_pass http://localhost:RPC_KUBO_PORT; + proxy_set_header Host $host; + } + location /api/v0/block/ { + proxy_pass http://localhost:RPC_KUBO_PORT; + proxy_set_header Host $host; + } + location /api/v0/pubsub/ { + proxy_pass http://localhost:RPC_KUBO_PORT; + proxy_set_header Host $host; + } +} diff --git a/doc/nginx/subdomain-ipfs-gateway.nginx.conf b/doc/nginx/subdomain-ipfs-gateway.nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..ddf83f888aa8c954b6d4c32ab48e72c5c718dd99 --- /dev/null +++ b/doc/nginx/subdomain-ipfs-gateway.nginx.conf @@ -0,0 +1,90 @@ +# redirects all http to https +server { + listen 80 ; + listen [::]:80 ; + server_name EXAMPLE.ORG; + return 301 https://$host$request_uri; +} +server { + listen 80 ; + listen [::]:80 ; + server_name *.ipns.EXAMPLE.ORG; + return 301 https://$host$request_uri; +} +server { + listen 80 ; + listen [::]:80 ; + server_name *.ipfs.EXAMPLE.ORG; + return 301 https://$host$request_uri; +} + +# serve homepage on root and allow gateway to perform /ipfs and /ipns redirections +server { + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/letsencrypt/live/EXAMPLE.ORG/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/EXAMPLE.ORG/privkey.pem; + + server_name EXAMPLE.ORG; + + access_log /var/log/nginx/EXAMPLE.ORG_access.log; + error_log /var/log/nginx/EXAMPLE.ORG_error.log; + + # only ipfs and ipns get redirected + location ~ ^/(ipfs|ipns)/ { + add_header X-Robots-Tag "noindex"; + proxy_pass http://localhost:KUBO_GATEWAY_PORT; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + } + + # other locations get redirected to fixed website + location / { + proxy_pass http://localhost:KUBO_GATEWAY_PORT; + proxy_set_header Host EXAMPLE-ORG.ipns.EXAMPLE.ORG; + proxy_set_header X-Forwarded-Proto https; + } +} + +# ipfs wildcard +server { + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/letsencrypt/live/ipfs.EXAMPLE.ORG/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/ipfs.EXAMPLE.ORG/privkey.pem; + + server_name *.ipfs.EXAMPLE.ORG; + + access_log /var/log/nginx/EXAMPLE.ORG_access.log; + error_log /var/log/nginx/EXAMPLE.ORG_error.log; + + location / { + add_header X-Robots-Tag "noindex"; + proxy_pass http://localhost:KUBO_GATEWAY_PORT; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + } +} + +# ipns wildcard +server { + listen 443 ssl; + listen [::]:443 ssl; + + ssl_certificate /etc/letsencrypt/live/ipns.EXAMPLE.ORG/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/ipns.EXAMPLE.ORG/privkey.pem; + + server_name *.ipns.EXAMPLE.ORG; + + access_log /var/log/nginx/EXAMPLE.ORG_access.log; + error_log /var/log/nginx/EXAMPLE.ORG_error.log; + + location / { + add_header X-Robots-Tag "noindex"; + proxy_pass http://localhost:KUBO_GATEWAY_PORT; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + } +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index bc433f3d6f701725a94db259fcb1a9ac3185b264..4adf11484c40883e6fa30818bb66967d488e45d3 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -83,19 +83,6 @@ services: # use the datapod collector and indexer, start using given IPNS entry command: ['./src/indexer/start.ts', '${DATAPOD_BOOT}'] - # ------ - gateway: - image: h30x/duniter-datapod - depends_on: - kubo: - condition: service_healthy - ports: - - ${SUBMIT_GATEWAY_LISTEN_PORT}:3000 - environment: - KUBO_RPC: 'http://kubo:5001' - restart: always - command: ['./src/gateway/start.ts'] - volumes: db_data: kubo_data: diff --git a/src/gateway/README.md b/src/gateway/README.md deleted file mode 100644 index 000031e6723ba6ba53f3790a0a60e2c8cc901977..0000000000000000000000000000000000000000 --- a/src/gateway/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Gateway - -This gateway is a way for non-p2p clients to submit data to a http endpoint that will then be forwarded to the p2p network. - -It is a stupid gateway that puts all content to local kubo node (but does not pin) and forwards the cid of the first block on pubsub topic. - -Since it does not include any protection mechanism because it relies on datapod network security, it is recommanded to limit the incoming network traffic and apply IP-based protections, preventing an attacker to trigger IPFS garbage collection too easily. Or to limit connection to trusted users. - -## Start - -Start the gateway with: - -```sh -pnpm exec tsx ./src/gateway/start.ts -``` - -## Sumbit data - -The data submitted to the gateway should be a JSON-encoded array of strings. - -- the first string is the CID of the index request -- the other strings are base64 blocks of all data (index request, data, images...) \ No newline at end of file diff --git a/src/gateway/start.ts b/src/gateway/start.ts deleted file mode 100644 index 8610def6c2b4ff7a9d54b73f83ce85d6ca120c0f..0000000000000000000000000000000000000000 --- a/src/gateway/start.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { TOPIC } from '../consts' -import { kubo } from '../kubo' -import { createServer } from 'http' -import type { IncomingMessage, ServerResponse } from 'http' - -const port = process.env.SUBMIT_GATEWAY_LISTEN_PORT || 3000 -const host = '127.0.0.1' -const postgatewayUrl = `http://${host}:${port}` -const GATEWAY_LANDING_PAGE = `<!DOCTYPE html> -<html> - <head> - <meta charset="UTF-8"> - <meta name="viewport" content="width=device-width, initial-scale=1"/> - <title>Datapod Gateway</title> - </head> - <body> - <h1>Datapod Gateway</h1> - <p>Use POST requests to submit data programatically. Alternatively, you can submit data manually in the form below for testing purpose.</p> - <form method="post" action="${postgatewayUrl}" enctype='text/plain'> - Blocks as base64 json list: - <input type="text" name="blocks" size=100 placeholder="['index request cid', 'index request as base64', 'data as base 64', 'optional aditionnal data as base64'...]"/> - <input type="submit" value="Submit" /> - </form> - </body> -</html> -` - -// request handler -function handleRequest(request: IncomingMessage, response: ServerResponse<IncomingMessage>) { - // use post requests to submit data - if (request.method == 'POST') { - let body = '' - request.on('data', function (data) { - body += data - }) - request.on('end', function () { - // hack to allow submitting data with form - if (body.startsWith('blocks=')) { - body = body.slice(7) - } - console.log('Body: ' + body) - try { - const blocks = JSON.parse(body) - let first = true - for (const b of blocks) { - // first item is the index request CID - if (first) { - kubo.pubsub.publish(TOPIC, new TextEncoder().encode(b + '\n')) - first = false - continue - } - // other items are blocks - const buffer = Buffer.from(b, 'base64') - const bytes = new Uint8Array(buffer) - kubo.block.put(bytes).then((cid) => {}) - } - response.writeHead(200, { 'Content-Type': 'text/html' }) - response.end('data received and forwarded<br/>' + JSON.stringify(blocks, null, 4)) - } catch (e: any) { - response.writeHead(400, { 'Content-Type': 'text/html' }) - response.end('invalid request\n' + e.toString()) - } - }) - } else { - var html = GATEWAY_LANDING_PAGE - response.writeHead(200, { 'Content-Type': 'text/html' }) - response.end(html) - } -} - -const server = createServer(handleRequest) -server.listen(port) -console.log(`Listening on ${postgatewayUrl}`)