diff --git a/.env.example b/.env.example index 101e7ed..a0c1769 100644 --- a/.env.example +++ b/.env.example @@ -1,17 +1,79 @@ +# The version of singularity you will use to make deals. Generally, this should not change SINGULARITY_TAG=v0.3.2 + +# Comma seperated list of storage providers motion should make storage deals with +# You must set this value to contain at least one storage provider for motion to +# work MOTION_STORAGE_PROVIDERS= -#MOTION_PRICE_PER_GIB_EPOCH= -#MOTION_PRICE_PER_GIB= -#MOTION_PRICE_PER_DEAL= -#MOTION_DEAL_START_DELAY= -#MOTION_DEAL_DURATION= -#MOTION_VERIFIED_DEAL= -#LOTUS_TEST= -#LOTUS_API= -#LOTUS_TOKEN= -#MOTION_SINGULARITY_MAX_CAR_SIZE= -#MOTION_SINGULARITY_PACK_THRESHOLD= -#MOTION_SINGULARITY_SCHEDULE_CRON= -#MOTION_SINGULARITY_SCHEDULE_DEAL_NUMBER= + +# The private key of the wallet you will use with motion, in hexadecimal format. +# This is the output of `lotus wallet export ~address~` if you are using lotus +# If you are obtaining a wallet through another method follow your wallet providers +# instructions to get your wallet's provider key MOTION_WALLET_KEY= -SINGULARITY_CONTENT_PROVIDER_DOMAIN= \ No newline at end of file + +# This is the domain/IP you will expose publicly to transfer data to storage providers +# When you initialize the singularity docker setup, it will start a server to deliver +# content to storage providers on localhost:7778. However, you will need to either +# open this specific port on your firewall, and set this value to http://~your-static-ip~:7778 +# or you will need to setup reverse proxying from a dedicated web server like NGinx +SINGULARITY_CONTENT_PROVIDER_DOMAIN= + +# Additional configuration parameters, sorted by the likelyhood you will want to set +# custom configuration values on your motion instance. In general, most of these should +# not need to change. + +# Frequency (as a cron tab expression) with which motion should attempt to make +# new Filecoin deals. Defaults to every minute +#MOTION_SINGULARITY_SCHEDULE_CRON="* * * * *" + +# Maximum number of deals motion should attempt to make when checking for new deals. +# Defaults to 1 +#MOTION_SINGULARITY_SCHEDULE_DEAL_NUMBER=1 + +# Specifies whether motion deals are made with FIL+ Data Cap allocation. In order to use this, you must apply for +# and receive a data cap allocation for your deals. In general, we recommend only those familiar with the Fil+ +# application process use FIL+, and instead contract out of band with storage providers to make non verified (non Fil+) +# deals. +#MOTION_VERIFIED_DEAL=false + +# ON-CHAIN price per gigabyte for motion deals +# In general, paid deals for motion should be negotiated in out of band contracts +# and there is no reason to change this value +#MOTION_PRICE_PER_GIB_EPOCH=0 + +# Another way to specify on-chain pricing for deals. There is no reason to change this value +#MOTION_PRICE_PER_GIB=0 + +# Another way to specify on-chain pricing for deals. There is no reason to change this value +#MOTION_PRICE_PER_DEAL=0 + +# Maximum delay time (in nanoseconds) between when a motion deal is proposed and when it's expected to be on chain. +# Defaults to 72 hours. In general, you should not change this unless your contract specifies something +# different. +#MOTION_DEAL_START_DELAY=259200000000000 + +# Initial total duration for motion deals stored on filecoin (specified in nanoseconds). +# Defaults to 1 year. In general, you should not change this unless your contracts specify something different +#MOTION_DEAL_DURATION=31536000000000000 + +# Specifies hether this operation of motion is running on mainnet or a testnet. +# Should be left to false unless you are a developer +#LOTUS_TEST=false + +# API endpoint to extract chain data from Lotus. You should not need to change this +#LOTUS_API=https://api.node.glif.io/rpc/v1 + +# Token for Lotus API. You do not need to specify a token unless you change the Lotus +# API endpoint +#LOTUS_TOKEN= + +# Maximum size of packed CAR files for deals made with singularity. +# Defaults to close to 32 gigabytes. You should not need to change this +#MOTION_SINGULARITY_MAX_CAR_SIZE=31.5GiB + +# Threshold for the minimum amount of data (in bytes) to trigger a motion +# deal on filecoin. Defaults to 16GiB. You should not need to change this +#MOTION_SINGULARITY_PACK_THRESHOLD=17179869184 + + diff --git a/README.md b/README.md index 0ed10e5..cced630 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,95 @@ # :motorcycle: motion +*Accelerating data onto Filecoin!* + +## Table of Contents + +- [Background](#background) +- [Install](#install-and-setup) +- [Usage](#usage) +- [API Specification](#api-specification) +- [Status](#status) +- [Local Development](#local-development) +- [License](#license) + +## Background + Motion is a service to propel data onto Filecoin network via a simple easy to use API. It aims to create an easy path for independent software vendors to integrate Filecoin as a storage layer. -## Usage +## Install and setup + +### Prerequisites + +1. A list of storage providers who have agreed to store data for you on Filecoin, with whom you've obtained contracts to specify terms of storage. Work with a Motion account manager to help set these up. If you are simply testing, your account manager can offer sample providers to test with. + +2. A Filecoin wallet to make deals with, for which you are in possession of the private key. Various options for obtaining a wallet can be found here (https://docs.filecoin.io/basics/assets/wallets/). Your Motion account manager may be able to offer a test wallet to use during a testing phase. + +3. A server (bare metal or VM) to run Motion on, with Docker Engine installed on that server with the Docker Compose plugin included. Recommended hardware requirements for servers are: +- Because we run docker, Linux variants are preferred for the OS +- Recommend at least >500GB disk space available for staging data. The complete Filecoin deal making process takes up to 3 days, and you will need to hold all data until deal making is complete. So the amount of free space you will need is roughly the amount of data you want to onboard per day times 3. +- In general, we do not believe Motion is processor or memory intensive, but a machine with at least 32GB of RAM is optimal +- You will also need `git` installed on this machine. +- The server must have a static IP and/or a domain name that points to it, with a port open so you can transfer data from Motion to Filecoin storage providers -```text -$ motion --help -NAME: - motion - Propelling data onto Filecoin - -USAGE: - motion [global options] command [command options] [arguments...] - -COMMANDS: - help, h Shows a list of commands or help for one command - -GLOBAL OPTIONS: - --dealDuration value The duration of deals made on Filecoin (default: One year (356 days)) [$MOTION_DEAL_DURATION] - --dealStartDelay value The deal start epoch delay. (default: 72 hours) [$MOTION_DEAL_START_DELAY] - --experimentalRemoteSingularityAPIUrl value When using a singularity as the storage engine, if set, uses a remote HTTP API to interface with Singularity (default: use singularity as a code library) - --experimentalSingularityContentURLTemplate value When using a singularity as the storage engine, if set, setups up online deals to use the given url template for making online deals (default: make offline deals) - --experimentalSingularityScheduleCron value When using a singularity as the storage engine, if set, setups up the cron schedule to send out batch deals. (default: disabled) - --experimentalSingularityScheduleDealNumber value When using a singularity as the storage engine, if set, setups up the max deal number per triggered schedule. (default: unlimited) - --experimentalSingularityStore Whether to use experimental Singularity store as the storage and deal making engine (default: Local storage is used) - --help, -h show help - --pricePerDeal value The maximum price per deal in attoFIL. (default: 0) [$MOTION_PRICE_PER_DEAL] - --pricePerGiB value The maximum price per GiB in attoFIL. (default: 0) [$MOTION_PRICE_PER_GIB] - --pricePerGiBEpoch value The maximum price per GiB per Epoch in attoFIL. (default: 0) [$MOTION_PRICE_PER_GIB_EPOCH] - --replicationFactor value The number of desired replicas per blob (default: Number of storage providers; see 'storageProvider' flag.) - --singularityMaxCarSize value The maximum Singularity generated CAR size (default: "31.5GiB") - --singularityPackThreshold value The Singularity store pack threshold in number of bytes (default: 17,179,869,184 (i.e. 16 GiB)) - --storageProvider value, --sp value [ --storageProvider value, --sp value ] Storage providers to which to make deals with. Multiple providers may be specified. (default: No deals are made to replicate data onto storage providers.) [$MOTION_STORAGE_PROVIDERS] - --storeDir value The path at which to store Motion data (default: OS Temporary directory) [$MOTION_STORE_DIR] - --walletKey value Hex encoded private key for the wallet to use with motion [$MOTION_WALLET_KEY] - - Lotus - - --lotus-test (default: false) [$LOTUS_TEST] - --lotusApi value Lotus RPC API endpoint (default: "https://api.node.glif.io/rpc/v1") [$LOTUS_API] - --lotusToken value Lotus RPC API token [$LOTUS_TOKEN] +### Setting up motion for the first time + +Start by cloning this repository: + +```shell +git clone https://github.com/filecoin-project/motion.git ``` -## Run Server Locally +Before you can run motion, you must configure it. First, from the motion directory, copy `.env.example` to `.env`: -### Prerequisites +```shell +cp .env.example .env +``` + +Now open `.env` and set the required values for your instance of motion. At minimum, you need to set the following values (excerpt from `.env.example` file): + +```yaml +# Comma seperated list of storage providers motion should make storage deals with +# You must set this value to contain at least one storage provider for motion to +# work +MOTION_STORAGE_PROVIDERS= + +# The private key of the wallet you will use with motion, in hexadecimal format. +# This is the output of `lotus wallet export ~address~` if you are using lotus +# If you are obtaining a wallet through another method follow your wallet providers +# instructions to get your wallet's provider key +MOTION_WALLET_KEY= + +# This is the domain/IP you will expose publicly to transfer data to storage providers +# When you initialize the singularity docker setup, it will start a server to deliver +# content to storage providers on localhost:7778. However, you will need to either +# open this specific port on your firewall, and set this value to http://~your-static-ip~:7778 +# or you will need to setup reverse proxying from a dedicated web server like NGinx +SINGULARITY_CONTENT_PROVIDER_DOMAIN= +``` + +Again, work with your Motion account manager to choose your storage providers, obtain a wallet, and for assistance setting up your server to expose a port for data transfers publicly on your server. -* Docker container runtime (or your favourite container runtime). The remainder of this README assumes `docker`. -* `curl` (or your favourite HTTP client). The reminder of this README assumes `curl` +As needed, you can also set additional values in your motion `.env`` file for more custom configurations. -### Start Motion API +## Usage -To start the motion API server run: +Once you've configured your `.env`, you're ready to start motion. From the local repository, simply run ```shell -docker run --rm -p 40080:40080 ghcr.io/filecoin-project/motion:main +docker compose up ``` -The above starts the Motion HTTP API exposed on default listen address: http://localhost:40080. -It uses a temporary directory to store blobs in a flat file format. +(if you don't want to see log messages directly in your terminal, run `docker compose up -d`) -### Store blobs +You'll know your motion is up an running when you see a message like this in the docker logs: + +``` +2023-09-07 17:49:57 motion-motion-1 | 2023-09-08T00:49:57.530Z INFO motion/api/server server/server.go:53 HTTP server started successfully. {"address": "[::]:40080"} +``` +Your copy of motion is now running. The Motion HTTP API is now running on port 40080 of your server's localhost. + +### Store blobs + To store an example blob, use the following `curl` command : ```shell echo "fish" | curl -X POST -H "Content-Type: application/octet-stream" -d @- http://localhost:40080/v0/blob @@ -69,6 +99,18 @@ The response should include a blob ID which you can then use the fetch the blob {"id":"ad7ef987-a932-495c-aa0c-7ffcabeda45f"} ``` +### Storing onto Filecoin + +Motion will begin saving data to Filecoin when it's holding at least 16GB of data that hasn't been backed up with a storage provider. + +If you want to test storing an actual filecoin deal, the following simple script will put about 20GB of random data into motion: + +```shell +for i in {0..20}; do; head -c 1000000000 /dev/urandom | curl -X POST --data-binary @- -H "Content-Type: application/octet-stream" http://localhost:40080/v0/blob; done +``` + +This should be enough to trigger at least 1 Filecoin deal being made from Motion + ### Retrieve a stored blob To retrieve a stored blob, send a `GET` request to the Motion API with the desired blob ID. @@ -85,20 +127,28 @@ fish Alternatively, you can browse the same URL in a web browser, which should prompt you to download the binary file. -### Configure local store directory +### Check the status of an uploaded blob -To configure the local storage directory, set `--storeDir` flag as an argument to the container. -This will override the directy path used by Motion to store files locally. -The path can be a [mounted volume](https://docs.docker.com/storage/volumes/), which then allows you to retain the stored files after the container -is restarted and restore them back. For example, the following command uses a directory named `store` in the current working directory mounted as a volume on Motion container at path `/store`: +In addition to retrieving data for a blob, you can also check the status of its storage on Filecoin: ```shell -docker run --rm -p 40080:40080 -v $PWD/store:/store ghcr.io/filecoin-project/motion:main --storeDir=/store +curl http://localhost:40080/v0/blob/ad7ef987-a932-495c-aa0c-7ffcabeda45f/status | jq . ``` +(`jq` being used to pretty print here -- make sure it's installed on your machine. you don't need to pipe to jq but the output will be more readable) -### Check the status of an uploaded blob - -Not yet implemented. +```json +{ + "id": "ad7ef987-a932-495c-aa0c-7ffcabeda45f", + "Replicas": [ + { + "provider": "f1234", + "status": "active", + "lastVerified": "2020-12-01T22:48:00Z", + "expiration": "2021-08-18T22:48:00Z" + } + ] +} +``` ## API Specification diff --git a/cmd/motion/main.go b/cmd/motion/main.go index 12f645e..c3f21d9 100644 --- a/cmd/motion/main.go +++ b/cmd/motion/main.go @@ -140,7 +140,7 @@ func main() { Name: "verifiedDeal", Usage: "whether deals made with motion should be verified deals", DefaultText: "Deals are verified", - Value: true, + Value: false, EnvVars: []string{"MOTION_VERIFIED_DEAL"}, }, &cli.StringFlag{ diff --git a/compose-local-dev.yml b/compose-local-dev.yml index f5c27e8..97378ea 100644 --- a/compose-local-dev.yml +++ b/compose-local-dev.yml @@ -13,9 +13,6 @@ services: depends_on: db: condition: service_healthy - networks: - - default - - devnet singularity_dataset_worker: extends: @@ -24,9 +21,6 @@ services: depends_on: db: condition: service_healthy - networks: - - default - - devnet singularity_deal_pusher: extends: @@ -35,9 +29,6 @@ services: depends_on: db: condition: service_healthy - networks: - - default - - devnet singularity_deal_tracker: extends: @@ -46,9 +37,6 @@ services: depends_on: db: condition: service_healthy - networks: - - default - - devnet singularity_content_provider: extends: @@ -57,9 +45,6 @@ services: depends_on: db: condition: service_healthy - networks: - - default - - devnet motion: build: . @@ -88,16 +73,10 @@ services: - motion-singularity-volume:/usr/src/app/storage depends_on: - singularity_api - networks: - - default - - devnet volumes: motion-singularity-volume: networks: default: - name: motion - devnet: - name: devnet - external: true \ No newline at end of file + name: motion \ No newline at end of file diff --git a/integration/singularity/options.go b/integration/singularity/options.go index 294071f..1143602 100644 --- a/integration/singularity/options.go +++ b/integration/singularity/options.go @@ -52,7 +52,8 @@ func newOptions(o ...Option) (*options, error) { packThreshold: 16 << 30, datasetName: "MOTION_DATASET", scheduleCronPerpetual: true, - verifiedDeal: true, + verifiedDeal: false, + keepUnsealed: true, ipniAnnounce: true, scheduleDealSize: "0", totalDealSize: "0",