Skip to content
Snippets Groups Projects
Commit 04352af7 authored by Hugo Trentesaux's avatar Hugo Trentesaux
Browse files

add nginx examples

and remove gateway
parent 7da2ff75
Branches
Tags
No related merge requests found
# example .env file for docker-compose.prod.yml # example .env file for docker-compose.prod.yml
# database configuration
DB_USER=postgres DB_USER=postgres
DB_PASSWORD=postgrespassword DB_PASSWORD=postgrespassword
DB_DATABASE=postgres DB_DATABASE=postgres
# hasura configuration
HASURA_LISTEN_PORT=8888 HASURA_LISTEN_PORT=8888
HASURA_GRAPHQL_ADMIN_SECRET=hasura_admin_secret HASURA_GRAPHQL_ADMIN_SECRET=hasura_admin_secret
KUBO_GATEWAY_PORT=8080
KUBO_DOMAIN=datapod.coinduf.eu # kubo configuration
KUBO_GATEWAY_DOMAIN=gateway.datapod.coinduf.eu KUBO_DOMAIN=datapod.coinduf.eu # domain (used in p2p peering)
KUBO_GATEWAY_SUBDOMAIN=pagu.re KUBO_GATEWAY_PORT=8080 # listen port of ipfs http gateway
SUBMIT_GATEWAY_LISTEN_PORT=3000 KUBO_GATEWAY_DOMAIN=gateway.datapod.coinduf.eu # domain for kubo reverse proxy ipfs http gateway reverse proxy
KUBO_GATEWAY_SUBDOMAIN=pagu.re # domain for subdomain gateway (provides origin isolation but requires wildcard)
# configure the node boot # configure the node boot
DATAPOD_BOOT=bafyreih4jspnqnsd4o3sdqv7c765uyylhtlh5majjw6aq6clilkq7tmqey DATAPOD_BOOT=/ipfs/bafyreih4jspnqnsd4o3sdqv7c765uyylhtlh5majjw6aq6clilkq7tmqey
\ No newline at end of file \ No newline at end of file
...@@ -27,7 +27,7 @@ This will pull preconfigured images for postgres/hasura, kubo and datapod. This ...@@ -27,7 +27,7 @@ This will pull preconfigured images for postgres/hasura, kubo and datapod. This
- start collecting from default IPNS - start collecting from default IPNS
- index to database - index to database
You can then do a simple proxy_pass to `HASURA_LISTEN_PORT` and `KUBO_GATEWAY_PORT`. You can then do a simple proxy_pass to `HASURA_LISTEN_PORT` and `KUBO_GATEWAY_PORT`. Read more on [install page](./doc/install.md).
## Dev ## Dev
......
# Install
To install a datapod, use the [`docker-compose.prod.yml`](../docker-compose.prod.yml) and [`.env.prod.example`](../.env.prod.example) files:
```sh
# edit env file
vim .env
# start services
docker compose up -d
# follow only datapod logs
docker compose logs -f datapod
```
The logs should give you:
```
# TODO
```
## Setup reverse proxy
Depending on what you want to expose publicly, use the following nginx config files examples.
- to expose graphql endpoint, see [`hasura.nginx.conf`](./nginx/hasura.nginx.conf)
- to expose ipfs http gateway, see [`ipfs-gateway.nginx.conf`](./nginx/ipfs-gateway.nginx.conf)
- to expose http gateway with subdomain, see [`subdomain-ipfs-gateway.nginx.conf`](./nginx/subdomain-ipfs-gateway.nginx.conf) (requires wildcard certificates)
- to expose rpc api, see [`ipfs-rpc.nginx.conf`](./nginx/ipfs-rpc.nginx.conf) (should not be done on main IPFS node)
A bit of explanation about what they are:
**graphql endpoint** Endpoint used for search capabilities.
**ipfs http gateway** A way to get IPFS content (image for example) through http (without embeding a node).
**subdomain gateway** Same as gateway, but provides origin isolation and root domain, for example for websites.
**rpc api** A way to "lend" an IPFS node to clients who do not embed one through a subset of the rpc API. This should be done on a different node than the one used by the datapod. It should also come with a rate limiter or some kind of protection mecanism.
\ No newline at end of file
# https graphql + wss subscriptions
# for HASURA_DOMAIN (replace by your domain name)
# on HASURA_LISTEN_PORT (as choosen in environment file)
# http redirection
server {
listen 80 ;
listen [::]:80 ;
server_name <HASURA_DOMAIN>;
return 301 https://$host$request_uri;
}
# listen on 443
server {
listen 443 ssl;
listen [::]:443 ssl;
# for example with letsencrypt
ssl_certificate /etc/letsencrypt/live/<HASURA_DOMAIN>/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/<HASURA_DOMAIN>/privkey.pem;
server_name <HASURA_DOMAIN>;
access_log /var/log/nginx/<HASURA_DOMAIN_access>.log;
error_log /var/log/nginx/<HASURA_DOMAIN_error>.log;
location / {
add_header X-Robots-Tag "noindex";
proxy_pass http://localhost:HASURA_LISTEN_PORT;
proxy_set_header Host $host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
# replace the following
# KUBO_GATEWAY_DOMAIN = domain of your ipfs http gateway
# KUBO_GATEWAY_PORT = port on which it is listening
server {
listen 80 ;
listen [::]:80 ;
server_name KUBO_GATEWAY_DOMAIN;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/letsencrypt/live/KUBO_GATEWAY_DOMAIN/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/KUBO_GATEWAY_DOMAIN/privkey.pem;
server_name KUBO_GATEWAY_DOMAIN;
access_log /var/log/nginx/KUBO_GATEWAY_DOMAIN_access.log;
error_log /var/log/nginx/KUBO_GATEWAY_DOMAIN_error.log;
# main location is gateway
location / {
add_header X-Robots-Tag "noindex";
proxy_pass http://localhost:KUBO_GATEWAY_PORT;
proxy_set_header Host $host;
}
}
# replace the following:
# RPC_KUBO_DOMAIN = the name under which expose your RPC API
# RPC_KUBO_PORT = the rpc api port (ususally 5001)
server {
listen 80 ;
listen [::]:80 ;
server_name RPC_KUBO_DOMAIN;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/letsencrypt/live/RPC_KUBO_DOMAIN/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/RPC_KUBO_DOMAIN/privkey.pem;
server_name RPC_KUBO_DOMAIN;
access_log /var/log/nginx/RPC_KUBO_DOMAIN_access.log;
error_log /var/log/nginx/RPC_KUBO_DOMAIN_error.log;
# main location
location / {
return 400;
}
location /api/v0/dag/ {
proxy_pass http://localhost:RPC_KUBO_PORT;
proxy_set_header Host $host;
}
location /api/v0/block/ {
proxy_pass http://localhost:RPC_KUBO_PORT;
proxy_set_header Host $host;
}
location /api/v0/pubsub/ {
proxy_pass http://localhost:RPC_KUBO_PORT;
proxy_set_header Host $host;
}
}
# redirects all http to https
server {
listen 80 ;
listen [::]:80 ;
server_name EXAMPLE.ORG;
return 301 https://$host$request_uri;
}
server {
listen 80 ;
listen [::]:80 ;
server_name *.ipns.EXAMPLE.ORG;
return 301 https://$host$request_uri;
}
server {
listen 80 ;
listen [::]:80 ;
server_name *.ipfs.EXAMPLE.ORG;
return 301 https://$host$request_uri;
}
# serve homepage on root and allow gateway to perform /ipfs and /ipns redirections
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/letsencrypt/live/EXAMPLE.ORG/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/EXAMPLE.ORG/privkey.pem;
server_name EXAMPLE.ORG;
access_log /var/log/nginx/EXAMPLE.ORG_access.log;
error_log /var/log/nginx/EXAMPLE.ORG_error.log;
# only ipfs and ipns get redirected
location ~ ^/(ipfs|ipns)/ {
add_header X-Robots-Tag "noindex";
proxy_pass http://localhost:KUBO_GATEWAY_PORT;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
}
# other locations get redirected to fixed website
location / {
proxy_pass http://localhost:KUBO_GATEWAY_PORT;
proxy_set_header Host EXAMPLE-ORG.ipns.EXAMPLE.ORG;
proxy_set_header X-Forwarded-Proto https;
}
}
# ipfs wildcard
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/letsencrypt/live/ipfs.EXAMPLE.ORG/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/ipfs.EXAMPLE.ORG/privkey.pem;
server_name *.ipfs.EXAMPLE.ORG;
access_log /var/log/nginx/EXAMPLE.ORG_access.log;
error_log /var/log/nginx/EXAMPLE.ORG_error.log;
location / {
add_header X-Robots-Tag "noindex";
proxy_pass http://localhost:KUBO_GATEWAY_PORT;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
}
}
# ipns wildcard
server {
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/letsencrypt/live/ipns.EXAMPLE.ORG/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/ipns.EXAMPLE.ORG/privkey.pem;
server_name *.ipns.EXAMPLE.ORG;
access_log /var/log/nginx/EXAMPLE.ORG_access.log;
error_log /var/log/nginx/EXAMPLE.ORG_error.log;
location / {
add_header X-Robots-Tag "noindex";
proxy_pass http://localhost:KUBO_GATEWAY_PORT;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Proto https;
}
}
...@@ -83,19 +83,6 @@ services: ...@@ -83,19 +83,6 @@ services:
# use the datapod collector and indexer, start using given IPNS entry # use the datapod collector and indexer, start using given IPNS entry
command: ['./src/indexer/start.ts', '${DATAPOD_BOOT}'] command: ['./src/indexer/start.ts', '${DATAPOD_BOOT}']
# ------
gateway:
image: h30x/duniter-datapod
depends_on:
kubo:
condition: service_healthy
ports:
- ${SUBMIT_GATEWAY_LISTEN_PORT}:3000
environment:
KUBO_RPC: 'http://kubo:5001'
restart: always
command: ['./src/gateway/start.ts']
volumes: volumes:
db_data: db_data:
kubo_data: kubo_data:
......
# Gateway
This gateway is a way for non-p2p clients to submit data to a http endpoint that will then be forwarded to the p2p network.
It is a stupid gateway that puts all content to local kubo node (but does not pin) and forwards the cid of the first block on pubsub topic.
Since it does not include any protection mechanism because it relies on datapod network security, it is recommanded to limit the incoming network traffic and apply IP-based protections, preventing an attacker to trigger IPFS garbage collection too easily. Or to limit connection to trusted users.
## Start
Start the gateway with:
```sh
pnpm exec tsx ./src/gateway/start.ts
```
## Sumbit data
The data submitted to the gateway should be a JSON-encoded array of strings.
- the first string is the CID of the index request
- the other strings are base64 blocks of all data (index request, data, images...)
\ No newline at end of file
import { TOPIC } from '../consts'
import { kubo } from '../kubo'
import { createServer } from 'http'
import type { IncomingMessage, ServerResponse } from 'http'
const port = process.env.SUBMIT_GATEWAY_LISTEN_PORT || 3000
const host = '127.0.0.1'
const postgatewayUrl = `http://${host}:${port}`
const GATEWAY_LANDING_PAGE = `<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Datapod Gateway</title>
</head>
<body>
<h1>Datapod Gateway</h1>
<p>Use POST requests to submit data programatically. Alternatively, you can submit data manually in the form below for testing purpose.</p>
<form method="post" action="${postgatewayUrl}" enctype='text/plain'>
Blocks as base64 json list:
<input type="text" name="blocks" size=100 placeholder="['index request cid', 'index request as base64', 'data as base 64', 'optional aditionnal data as base64'...]"/>
<input type="submit" value="Submit" />
</form>
</body>
</html>
`
// request handler
function handleRequest(request: IncomingMessage, response: ServerResponse<IncomingMessage>) {
// use post requests to submit data
if (request.method == 'POST') {
let body = ''
request.on('data', function (data) {
body += data
})
request.on('end', function () {
// hack to allow submitting data with form
if (body.startsWith('blocks=')) {
body = body.slice(7)
}
console.log('Body: ' + body)
try {
const blocks = JSON.parse(body)
let first = true
for (const b of blocks) {
// first item is the index request CID
if (first) {
kubo.pubsub.publish(TOPIC, new TextEncoder().encode(b + '\n'))
first = false
continue
}
// other items are blocks
const buffer = Buffer.from(b, 'base64')
const bytes = new Uint8Array(buffer)
kubo.block.put(bytes).then((cid) => {})
}
response.writeHead(200, { 'Content-Type': 'text/html' })
response.end('data received and forwarded<br/>' + JSON.stringify(blocks, null, 4))
} catch (e: any) {
response.writeHead(400, { 'Content-Type': 'text/html' })
response.end('invalid request\n' + e.toString())
}
})
} else {
var html = GATEWAY_LANDING_PAGE
response.writeHead(200, { 'Content-Type': 'text/html' })
response.end(html)
}
}
const server = createServer(handleRequest)
server.listen(port)
console.log(`Listening on ${postgatewayUrl}`)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment