diff --git a/.travis.yml b/.travis.yml index 641f666..7f60a84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,61 +2,32 @@ language: go go: - 1.13.x sudo: false +services: + - redis-server matrix: include: - # Using vendored dependencies - - install: - - go get -u golang.org/x/lint/golint - - go get -u golang.org/x/tools/cmd/... - script: - - go mod download - - go test -v -race $(go list ./...) - - go vet $(go list ./...) - - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "") - - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1; done) - - go install github.com/chihaya/chihaya/cmd/chihaya - - chihaya --config=example_config.yaml --debug& - - pid=$! - - sleep 2 # wait for chihaya to start up (gross) - - chihaya e2e --debug - - kill $pid - # Using HEAD of dependencies - - install: - - go get -t -u ./... - - go get -u golang.org/x/lint/golint - - go get -u golang.org/x/tools/cmd/... - script: - - go test -v -race $(go list ./...) - - go vet $(go list ./...) - - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "") - - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1; done) - - go install github.com/chihaya/chihaya/cmd/chihaya - - chihaya --config=example_config.yaml --debug& - - pid=$! - - sleep 2 # wait for chihaya to start up (gross) - - chihaya e2e --debug - - kill $pid + # Using vendored dependencies + - install: + - ./dist/travis/install_reproducible.sh + script: + - ./dist/travis/check.sh + + # Using HEAD of dependencies + - install: + - ./dist/travis/install_tip.sh + script: + - ./dist/travis/check.sh + allow_failures: - # Using HEAD of dependencies - - install: - - go get -t -u ./... - - go get -u golang.org/x/lint/golint - - go get -u golang.org/x/tools/cmd/... - script: - - go test -v -race $(go list ./...) - - go vet $(go list ./...) - - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "") - - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1; done) - - go install github.com/chihaya/chihaya/cmd/chihaya - - chihaya --config=example_config.yaml --debug& - - pid=$! - - sleep 2 # wait for chihaya to start up (gross) - - chihaya e2e --debug - - kill $pid + # Using HEAD of dependencies + - install: + - ./dist/travis/install_tip.sh + script: + - ./dist/travis/check.sh notifications: irc: channels: - - irc.freenode.net#chihaya + - irc.freenode.net#chihaya use_notice: true skip_join: true on_success: always diff --git a/Dockerfile b/Dockerfile index fe62383..df2cbda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /go/src/github.com/chihaya/chihaya COPY . /go/src/github.com/chihaya/chihaya # Install our golang dependencies and compile our binary. -RUN CGO_ENABLED=0 GO111MODULE=on go install github.com/chihaya/chihaya/cmd/... +RUN CGO_ENABLED=0 go install github.com/chihaya/chihaya/cmd/... FROM alpine:latest RUN apk add --no-cache ca-certificates diff --git a/README.md b/README.md index 577de30..7d2a50c 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ In order to compile the project, the [latest stable version of Go] and knowledge ```sh $ git clone git@github.com:chihaya/chihaya.git $ cd chihaya -$ GO111MODULE=on go build ./cmd/chihaya +$ go build ./cmd/chihaya $ ./chihaya --help ``` @@ -84,6 +84,19 @@ Removing `-bench` will just run unit tests. $ go test -bench $(go list ./...) ``` +The Chihaya executable contains a command to end-to-end test a BitTorrent tracker. +See + +```sh +$ chihaya --help +``` + +### Configuration + +Configuration of Chihaya is done via one YAML configuration file. +The `dist/` directory contains an example configuration file. +Files and directories under `docs/` contain detailed information about configuring middleware, storage implementations, architecture etc. + ## Related projects - [BitTorrent.org](https://github.com/bittorrent/bittorrent.org): a static website containing the BitTorrent spec and all BEPs diff --git a/example_config.yaml b/dist/example_config.yaml similarity index 97% rename from example_config.yaml rename to dist/example_config.yaml index beae654..685c18e 100644 --- a/example_config.yaml +++ b/dist/example_config.yaml @@ -129,14 +129,15 @@ chihaya: # # This balances between # # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) # # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). - # gc_interval: 14m + # gc_interval: 3m - # # The frequency which metrics are pushed into a local Prometheus endpoint. + # # The interval at which metrics about the number of infohashes and peers + # # are collected and posted to Prometheus. # prometheus_reporting_interval: 1s # # The amount of time until a peer is considered stale. # # To avoid churn, keep this slightly larger than `announce_interval` - # peer_lifetime: 16m + # peer_lifetime: 31m # # The address of redis storage. # redis_broker: "redis://pwd@127.0.0.1:6379/0" diff --git a/dist/travis/check.sh b/dist/travis/check.sh new file mode 100755 index 0000000..774092a --- /dev/null +++ b/dist/travis/check.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + +go test -v -race $(go list ./...) +go vet $(go list ./...) +diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "") +(for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1; done) +go install github.com/chihaya/chihaya/cmd/chihaya + +# Run e2e test with example config. +chihaya --config=./dist/travis/config_memory.yaml --debug& +pid=$! +sleep 2 # wait for Chihaya to start up (gross) +chihaya e2e --debug +kill $pid + +# Run e2e test with redis. +chihaya --config=./dist/travis/config_redis.yaml --debug& +pid=$! +sleep 2 # wait for Chihaya to start up (gross) +chihaya e2e --debug +kill $pid \ No newline at end of file diff --git a/dist/travis/config_memory.yaml b/dist/travis/config_memory.yaml new file mode 100644 index 0000000..96e7d1a --- /dev/null +++ b/dist/travis/config_memory.yaml @@ -0,0 +1,42 @@ +# This config file is used by travis for end-to-end testing. +# See example_config.yaml for a commented, more complete configuration! + +chihaya: + announce_interval: 30m + min_announce_interval: 15m + prometheus_addr: "0.0.0.0:6880" + + http: + addr: "0.0.0.0:6969" + https_addr: "" + tls_cert_path: "" + tls_key_path: "" + read_timeout: 5s + write_timeout: 5s + enable_keepalive: false + idle_timeout: 30s + enable_request_timing: false + enable_legacy_php_urls: false + allow_ip_spoofing: false + real_ip_header: "x-real-ip" + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + udp: + addr: "0.0.0.0:6969" + max_clock_skew: 10s + private_key: "paste a random string here that will be used to hmac connection IDs" + enable_request_timing: false + allow_ip_spoofing: false + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + storage: + name: memory + config: + gc_interval: 3m + peer_lifetime: 31m + shard_count: 1024 + prometheus_reporting_interval: 1s diff --git a/dist/travis/config_redis.yaml b/dist/travis/config_redis.yaml new file mode 100644 index 0000000..5e05708 --- /dev/null +++ b/dist/travis/config_redis.yaml @@ -0,0 +1,45 @@ +# This config file is used by travis for end-to-end testing. +# See example_config.yaml for a commented, more complete configuration! + +chihaya: + announce_interval: 30m + min_announce_interval: 15m + prometheus_addr: "0.0.0.0:6880" + + http: + addr: "0.0.0.0:6969" + https_addr: "" + tls_cert_path: "" + tls_key_path: "" + read_timeout: 5s + write_timeout: 5s + enable_keepalive: false + idle_timeout: 30s + enable_request_timing: false + enable_legacy_php_urls: false + allow_ip_spoofing: false + real_ip_header: "x-real-ip" + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + udp: + addr: "0.0.0.0:6969" + max_clock_skew: 10s + private_key: "paste a random string here that will be used to hmac connection IDs" + enable_request_timing: false + allow_ip_spoofing: false + max_numwant: 100 + default_numwant: 50 + max_scrape_infohashes: 50 + + storage: + name: redis + config: + gc_interval: 3m + peer_lifetime: 31m + prometheus_reporting_interval: 1s + redis_broker: "redis://127.0.0.1:6379/0" + redis_read_timeout: 15s + redis_write_timeout: 15s + redis_connect_timeout: 15s \ No newline at end of file diff --git a/dist/travis/install_reproducible.sh b/dist/travis/install_reproducible.sh new file mode 100755 index 0000000..4570135 --- /dev/null +++ b/dist/travis/install_reproducible.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +# Install golint and go vet. +go get -u golang.org/x/lint/golint +go get -u golang.org/x/tools/cmd/... + +go mod download \ No newline at end of file diff --git a/dist/travis/install_tip.sh b/dist/travis/install_tip.sh new file mode 100755 index 0000000..3bbec68 --- /dev/null +++ b/dist/travis/install_tip.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -ex + +# Install golint and go vet. +go get -u golang.org/x/lint/golint +go get -u golang.org/x/tools/cmd/... + +go get -t -u ./... \ No newline at end of file diff --git a/docs/storage/redis.md b/docs/storage/redis.md index 25c70bc..da3b3fb 100644 --- a/docs/storage/redis.md +++ b/docs/storage/redis.md @@ -1,12 +1,20 @@ # Redis Storage -This storage system separates chihaya server from storage service, chihaya server achieves HA by storing all peer data in Redis, the storage service gets HA by creating cluster. If one chihaya node goes down, peer data will still be available in Redis service. +This storage implementation separates Chihaya from its storage service. +Chihaya achieves HA by storing all peer data in Redis. +Multiple instances of Chihaya can use the same redis instance concurrently. +The storage service can get HA by clustering. +If one instance of Chihaya goes down, peer data will still be available in Redis. -The HA of storage service is not considered here, it's another topic. In case Redis service is a single node, peer data will be unavailable if the node is down. So you should setup a Redis cluster for chihaya server in production. +The HA of storage service is not considered here. +In case Redis runs as a single node, peer data will be unavailable if the node is down. +You should consider setting up a Redis cluster for Chihaya in production. + +This storage implementation is currently orders of magnitude slower than the in-memory implementation. ## Use Case -When one chihaya instance is down, the Redis can continuily serve peer data through other chihaya instances. +When one instance of Chihaya is down, other instances can continue serving peers from Redis. ## Configuration @@ -16,14 +24,18 @@ chihaya: name: redis config: # The frequency which stale peers are removed. - gc_interval: 14m + # This balances between + # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value) + # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value). + gc_interval: 3m - # The frequency which metrics are pushed into a local Prometheus endpoint. + # The interval at which metrics about the number of infohashes and peers + # are collected and posted to Prometheus. prometheus_reporting_interval: 1s # The amount of time until a peer is considered stale. # To avoid churn, keep this slightly larger than `announce_interval` - peer_lifetime: 16m + peer_lifetime: 31m # The address of redis storage. redis_broker: "redis://pwd@127.0.0.1:6379/0" @@ -40,11 +52,12 @@ chihaya: ## Implementation -Seeders and Leechers for a particular InfoHash are stored with a redis hash structure, the infohash is used as hash key, peer key is field, last modified time is value. +Seeders and Leechers for a particular InfoHash are stored within a redis hash. +The InfoHash is used as key, _peer keys_ are the fields, last modified times are values. +Peer keys are derived from peers and contain Peer ID, IP, and Port. +All the InfoHashes (swarms) are also stored in a redis hash, with IP family as the key, infohash as field, and last modified time as value. -All the InfoHashes (swarms) are also stored into redis hash, IP family is the key, infohash is field, last modified time is value. - -Here is an example +Here is an example: ``` - IPv4 @@ -61,9 +74,8 @@ Here is an example ``` -In this case, prometheus will record two swarms, three seeders and one leecher. - -So tree keys are used to record the count of swarms, seeders and leechers for each group (IPv4, IPv6). +In this case, prometheus would record two swarms, three seeders, and one leecher. +These three keys per address family are used to record the count of swarms, seeders, and leechers. ``` - IPv4_infohash_count: 2 @@ -71,4 +83,5 @@ So tree keys are used to record the count of swarms, seeders and leechers for ea - IPv4_L_count: 1 ``` -Note: IPv4_infohash_count has the different meaning with `memory` storage, it represents the number of infohashes reported by seeder. +Note: IPv4_infohash_count has a different meaning compared to the `memory` storage: +It represents the number of infohashes reported by seeder, meaning that infohashes without seeders are not counted.