commit eee2810da6adb56eef26ddc6676513485721abf2 Author: Jimmy Zelinskie Date: Fri Jun 21 19:31:32 2013 -0400 initial diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c0a437d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +language: go + +services: + - redis-server diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..f8872e2 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +# This is the official list of Chihaya authors for copyright purposes. + +Jimmy Zelinskie +Justin Li diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..2a7fed2 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,5 @@ +# This is the official list of Chihaya contributors. + +Jimmy Zelinskie +Justin Li +Kaleb Elwert diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4e2eca7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Chihaya is released under a BSD 2-Clause license, reproduced below. + +Copyright (c) 2013, The Chihaya Authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..75a6989 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +chihaya +======= + +[![Build Status](https://travis-ci.org/jzelinskie/chihaya.png?branch=master)](https://travis-ci.org/jzelinskie/chihaya) + +chihaya is a high-performance [BitTorrent tracker](http://en.wikipedia.org/wiki/BitTorrent_tracker) written in the Go programming language. +It isn't quite ready for prime-time just yet, but these are the features that it'll have: + +- Low processing and memory footprint +- IPv6 support +- Support for multiple storage backends +- Linear horizontal scalability (depending on the backends) + + +Installing +---------- + + $ go install github.com/jzelinskie/chihaya + + +Configuration +------------- + +Configuration is done in a JSON formatted file specified with the -config flag. +One can start with `example/config.json`, as a base. Check out GoDoc for more info. + + +Contributing +------------ + +If you want to make a smaller change, just go ahead and do it, and when you're +done send a pull request through GitHub. If there's a larger change you want to +make, it would be preferable to discuss it first via a GitHub issue or by +getting in touch on IRC. Always remember to gofmt your code! + + +Contact +------- + +If you have any questions or want to contribute something, come say hi in the +IRC channel: **#chihaya on [freenode](http://freenode.net/)** +([webchat](http://webchat.freenode.net?channels=chihaya)). + diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..842c960 --- /dev/null +++ b/config/config.go @@ -0,0 +1,62 @@ +package config + +import ( + "encoding/json" + "os" + "time" +) + +type Config struct { + Addr string `json:"addr"` + Storage Storage `json:"storage"` + Private bool `json:"private"` + Freeleech bool `json:"freeleech"` + + Announce Duration `json:"announce"` + MinAnnounce Duration `json:"min_announce"` + BufferPoolSize int `json:"bufferpool_size"` + + Whitelist []string `json:"whitelist"` +} + +// StorageConfig represents the settings used for storage or cache. +type Storage struct { + Driver string `json:"driver"` + Protocol string `json:"protocol"` + Addr string `json:"addr"` + Username string `json:"user"` + Password string `json:"pass"` + Schema string `json:"schema"` + Encoding string `json:"encoding"` +} + +type Duration struct { + time.Duration +} + +func (d *Duration) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + d.Duration, err = time.ParseDuration(str) + return err +} + +func New(path string) (*Config, error) { + expandedPath := os.ExpandEnv(path) + f, err := os.Open(expandedPath) + if err != nil { + return nil, err + } + defer f.Close() + + conf := &Config{} + err = json.NewDecoder(f).Decode(conf) + if err != nil { + return nil, err + } + return conf, nil +} diff --git a/example/config.json b/example/config.json new file mode 100644 index 0000000..991e891 --- /dev/null +++ b/example/config.json @@ -0,0 +1,19 @@ +{ + + "addr": ":34000" + "announce": "30m", + "min_announce": "15m", + "freelech": false, + "private": true, + "bufferpool_size": 500, + + "storage": { + "driver": "redis", + "addr": "127.0.0.1:6379", + "user": "root", + "pass": "", + }, + + "whitelist": [], + +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..90e2297 --- /dev/null +++ b/main.go @@ -0,0 +1,75 @@ +// Copyright 2013 The Chihaya Authors. All rights reserved. +// Use of this source code is governed by the BSD 2-Clause license, +// which can be found in the LICENSE file. + +package main + +import ( + "flag" + "log" + "os" + "os/signal" + "runtime" + "runtime/pprof" + + "github.com/jzelinskie/chihaya/config" + "github.com/jzelinskie/chihaya/server" +) + +var ( + profile bool + configFile string +) + +func init() { + flag.BoolVar(&profile, "profile", false, "Generate profiling data for pprof into chihaya.cpu") + flag.StringVar(&configFile, "config", "", "The location of a valid configuration file.") +} + +func main() { + flag.Parse() + runtime.GOMAXPROCS(runtime.NumCPU()) + + if configFile != "" { + conf, err := config.Parse(configFile) + if err != nil { + log.Fatalf("Failed to parse configuration file: %s\n", err) + } + } + + if profile { + log.Println("Running with profiling enabled") + f, err := os.Create("chihaya.cpu") + if err != nil { + log.Fatalf("Failed to create profile file: %s\n", err) + } + defer f.Close() + pprof.StartCPUProfile(f) + } + + s := server.New(conf) + + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + + if profile { + pprof.StopCPUProfile() + } + + log.Println("Caught interrupt, shutting down..") + err := s.Stop() + if err != nil { + panic("Failed to shutdown cleanly") + } + log.Println("Shutdown successfully") + <-c + os.Exit(0) + }() + + err = s.Start() + if err != nil { + log.Fatalf("Failed to start server: %s\n", err) + } +} diff --git a/server/announce.go b/server/announce.go new file mode 100644 index 0000000..426f69f --- /dev/null +++ b/server/announce.go @@ -0,0 +1,81 @@ +package server + +import ( + "bytes" + "errors" + "log" + + "github.com/jzelinskie/chihaya/config" + "github.com/jzelinskie/chihaya/storage" +) + +func (h *handler) serveAnnounce(w *http.ResponseWriter, r *http.Request) { + buf := h.bufferpool.Take() + defer h.bufferpool.Give(buf) + defer h.writeResponse(&w, r, buf) + + user, err := validatePasskey(dir, h.storage) + if err != nil { + fail(err, buf) + return + } + + pq, err := parseQuery(r.URL.RawQuery) + if err != nil { + fail(errors.New("Error parsing query"), buf) + return + } + + ip, err := determineIP(r, pq) + if err != nil { + fail(err, buf) + return + } + + err := validateParsedQuery(pq) + if err != nil { + fail(errors.New("Malformed request"), buf) + return + } + + if !whitelisted(peerId, h.conf) { + fail(errors.New("Your client is not approved"), buf) + return + } + + torrent, exists, err := h.storage.FindTorrent(infohash) + if err != nil { + panic("server: failed to find torrent") + } + if !exists { + fail(errors.New("This torrent does not exist"), buf) + return + } + + if torrent.Status == 1 && left == 0 { + err := h.storage.UnpruneTorrent(torrent) + if err != nil { + panic("server: failed to unprune torrent") + } + torrent.Status = 0 + } else if torrent.Status != 0 { + fail( + fmt.Errorf( + "This torrent does not exist (status: %d, left: %d)", + torrent.Status, + left, + ), + buf, + ) + return + } + + //go +} + +func whitelisted(peerId string, conf config.Config) bool { + // TODO Decide if whitelist should be in storage or config +} + +func newPeer() { +} diff --git a/server/query.go b/server/query.go new file mode 100644 index 0000000..35f2bb4 --- /dev/null +++ b/server/query.go @@ -0,0 +1,119 @@ +package server + +import ( + "errors" + "strconv" +) + +type parsedQuery struct { + infohashes []string + params map[string]string +} + +func (pq *parsedQuery) getUint64(key string) (uint64, bool) { + str, exists := pq[key] + if !exists { + return 0, false + } + val, err := strconv.Uint64(str, 10, 64) + if err != nil { + return 0, false + } + return val, true +} + +func parseQuery(query string) (*parsedQuery, error) { + var ( + keyStart, keyEnd int + valStart, valEnd int + firstInfohash string + + onKey = true + hasInfohash = false + + pq = &parsedQuery{ + infohashes: nil, + params: make(map[string]string), + } + ) + + for i, length := 0, len(query); i < length; i++ { + separator := query[i] == '&' || query[i] == ';' || query[i] == '?' + if separator || i == length-1 { + if onKey { + keyStart = i + 1 + continue + } + + if i == length-1 && !separator { + if query[i] == '=' { + continue + } + valEnd = i + } + + keyStr, err := url.QueryUnescape(query[keyStart : keyEnd+1]) + if err != nil { + return err + } + valStr, err := url.QueryUnescape(query[valStart : valEnd+1]) + if err != nil { + return err + } + + pq.params[keyStr] = valStr + + if keyStr == "info_hash" { + if hasInfohash { + // Multiple infohashes + if pq.infohashes == nil { + pq.infohashes = []string{firstInfoHash} + } + pq.infohashes = append(pq.infohashes, valStr) + } else { + firstInfohash = valStr + hasInfohash = true + } + } + + onKey = true + keyStart = i + 1 + } else if query[i] == '=' { + onKey = false + valStart = i + 1 + } else if onKey { + keyEnd = i + } else { + valEnd = i + } + } + return +} + +func validateParsedQuery(pq *parsedQuery) error { + infohash, ok := pq["info_hash"] + if infohash == "" { + return errors.New("infohash does not exist") + } + peerId, ok := pq["peer_id"] + if peerId == "" { + return errors.New("peerId does not exist") + } + port, ok := pq.getUint64("port") + if ok == false { + return errors.New("port does not exist") + } + uploaded, ok := pq.getUint64("uploaded") + if ok == false { + return errors.New("uploaded does not exist") + } + downloaded, ok := pq.getUint64("downloaded") + if ok == false { + return errors.New("downloaded does not exist") + } + left, ok := pq.getUint64("left") + if ok == false { + return errors.New("left does not exist") + } + return nil +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..5d4af4f --- /dev/null +++ b/server/server.go @@ -0,0 +1,155 @@ +package server + +import ( + "bytes" + "errors" + "net" + "net/http" + "path" + "strconv" + "sync" + "sync/atomic" + + "github.com/jzelinskie/bufferpool" + + "github.com/jzelinskie/chihaya/config" + "github.com/jzelinskie/chihaya/storage" +) + +type Server struct { + http.Server + listener *net.Listener +} + +func New(conf *config.Config) { + return &Server{ + Addr: conf.Addr, + Handler: newHandler(conf), + } +} + +func (s *Server) Start() error { + s.listener, err = net.Listen("tcp", config.Addr) + if err != nil { + return err + } + s.Handler.terminated = false + s.Serve(s.listener) + s.Handler.waitgroup.Wait() + s.Handler.storage.Shutdown() + return nil +} + +func (s *Server) Stop() error { + s.Handler.waitgroup.Wait() + s.Handler.terminated = true + return s.Handler.listener.Close() +} + +type handler struct { + bufferpool *bufferpool.BufferPool + conf *config.Config + deltaRequests int64 + storage *storage.Storage + terminated bool + waitgroup sync.WaitGroup +} + +func newHandler(conf *config.Config) { + return &Handler{ + bufferpool: bufferpool.New(conf.BufferPoolSize, 500), + conf: conf, + storage: storage.New(&conf.Storage), + } +} + +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if h.terminated { + return + } + + h.waitgroup.Add(1) + defer h.waitgroup.Done() + + if r.URL.Path == "/stats" { + h.serveStats(&w, r) + return + } + + dir, action := path.Split(requestPath) + switch action { + case "announce": + h.serveAnnounce(&w, r) + return + case "scrape": + // TODO + h.serveScrape(&w, r) + return + default: + buf := h.bufferpool.Take() + fail(errors.New("Unknown action"), buf) + h.writeResponse(&w, r, buf) + return + } +} + +func writeResponse(w *http.ResponseWriter, r *http.Request, buf *bytes.Buffer) { + r.Close = true + w.Header().Add("Content-Type", "text/plain") + w.Header().Add("Connection", "close") + w.Header().Add("Content-Length", strconv.Itoa(buf.Len())) + w.Write(buf.Bytes()) + w.(http.Flusher).Flush() + atomic.AddInt64(h.deltaRequests, 1) +} + +func fail(err error, buf *bytes.Buffer) { + buf.WriteString("d14:failure reason") + buf.WriteString(strconv.Itoa(len(err))) + buf.WriteRune(':') + buf.WriteString(err) + buf.WriteRune('e') +} + +func validatePasskey(dir string, s *storage.Storage) (storage.User, error) { + if len(dir) != 34 { + return nil, errors.New("Your passkey is invalid") + } + passkey := dir[1:33] + + user, exists, err := s.FindUser(passkey) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.New("Passkey not found") + } + + return user, nil +} + +func determineIP(r *http.Request, pq *parsedQuery) (string, error) { + ip, ok := pq.params["ip"] + if !ok { + ip, ok = pq.params["ipv4"] + if !ok { + ips, ok := r.Header["X-Real-Ip"] + if ok && len(ips) > 0 { + ip = ips[0] + } else { + portIndex := len(r.RemoteAddr) - 1 + for ; portIndex >= 0; portIndex-- { + if r.RemoteAddr[portIndex] == ':' { + break + } + } + if portIndex != -1 { + ip = r.RemoteAddr[0:portIndex] + } else { + return "", errors.New("Failed to parse IP address") + } + } + } + } + return &ip, nil +} diff --git a/storage/data.go b/storage/data.go new file mode 100644 index 0000000..8ff1851 --- /dev/null +++ b/storage/data.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Chihaya Authors. All rights reserved. +// Use of this source code is governed by the BSD 2-Clause license, +// which can be found in the LICENSE file. + +package storage + +type Peer struct { + Id string + UserId uint64 + TorrentId uint64 + + Port uint + Ip string + Addr []byte + + Uploaded uint64 + Downloaded uint64 + Left uint64 + Seeding bool + + StartTime int64 // Unix Timestamp + LastAnnounce int64 +} + +type Torrent struct { + Id uint64 + InfoHash string + UpMultiplier float64 + DownMultiplier float64 + + Seeders map[string]*Peer + Leechers map[string]*Peer + + Snatched uint + Status int64 + LastAction int64 +} + +type User struct { + Id uint64 + Passkey string + UpMultiplier float64 + DownMultiplier float64 + Slots int64 + UsedSlots int64 + + SlotsLastChecked int64 +} diff --git a/storage/redis/redis.go b/storage/redis/redis.go new file mode 100644 index 0000000..1420383 --- /dev/null +++ b/storage/redis/redis.go @@ -0,0 +1,5 @@ +package redis + +import ( + "github.com/jzelinskie/chihaya/storage" +) diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..cbc0ceb --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,65 @@ +package storage + +import ( + "fmt" + + "github.com/jzelinskie/chihaya/config" +) + +var drivers = make(map[string]StorageDriver) + +type StorageDriver interface { + New(*config.StorageConfig) (Storage, error) +} + +func Register(name string, driver StorageDriver) { + if driver == nil { + panic("storage: Register driver is nil") + } + if _, dup := drivers[name]; dup { + panic("storage: Register called twice for driver " + name) + } + drivers[name] = driver +} + +func New(name string, conf *config.Storage) (Storage, error) { + driver, ok := drivers[name] + if !ok { + return nil, fmt.Errorf( + "storage: unknown driver %q (forgotten import?)", + name, + ) + } + store, err := driver.New(conf) + if err != nil { + return nil, err + } + return store, nil +} + +type Storage interface { + Shutdown() error + + FindUser(passkey []byte) (*User, bool, error) + FindTorrent(infohash []byte) (*Torrent, bool, error) + UnpruneTorrent(torrent *Torrent) error + + RecordUser( + user *User, + rawDeltaUpload int64, + rawDeltaDownload int64, + deltaUpload int64, + deltaDownload int64, + ) error + RecordSnatch(peer *Peer, now int64) error + RecordTorrent(torrent *Torrent, deltaSnatch uint64) error + RecordTransferIP(peer *Peer) error + RecordTransferHistory( + peer *Peer, + rawDeltaUpload int64, + rawDeltaDownload int64, + deltaTime int64, + deltaSnatch uint64, + active bool, + ) error +}