diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 849a6b4..fc49d47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ jobs: arch: ppc64 - os: linux arch: ppc64le + - os: darwin + arch: arm64 exclude: - os: darwin arch: 386 @@ -68,7 +70,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: ^1.15 + go-version: ^1.16 - name: Build fat binary id: builder @@ -128,24 +130,19 @@ jobs: path: archive-is # Put files to archive.is directory - name: Create Release - uses: actions/create-release@v1 + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} body: | + See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/${{ github.sha }}/CHANGELOG.md). + **Digests in this release:** ``` ${{ needs.checksum.outputs.digest }} ``` - draft: false - prerelease: true - - - name: Upload release assets - uses: fnkr/github-action-ghr@v1 - if: startsWith(github.ref, 'refs/tags/') - env: - GHR_PATH: archive-is/ - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + draft: true + files: | + archive-is/* diff --git a/Makefile b/Makefile index a8338fb..281f8fc 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,7 @@ export GOPROXY = https://proxy.golang.org NAME = archive.is BINDIR ?= ./build/binary PACKDIR ?= ./build/package -LDFLAGS := $(shell echo "-X 'archive.is/version.Version=`git describe --tags --abbrev=0`'") -LDFLAGS := $(shell echo "${LDFLAGS} -X 'archive.is/version.Commit=`git rev-parse --short HEAD`'") -LDFLAGS := $(shell echo "${LDFLAGS} -X 'archive.is/version.BuildDate=`date +%FT%T%z`'") +LDFLAGS := $(shell echo "-X 'github.com/wabarc/archive.is.Version=`git describe --tags --abbrev=0`'") GOBUILD ?= CGO_ENABLED=0 go build -trimpath --ldflags "-s -w ${LDFLAGS} -buildid=" -v VERSION ?= $(shell git describe --tags `git rev-list --tags --max-count=1` | sed -e 's/v//g') GOFILES ?= $(wildcard ./cmd/archive.is/*.go) @@ -15,6 +13,7 @@ PACKAGES ?= $(shell go list ./...) PLATFORM_LIST = \ darwin-amd64 \ + darwin-arm64 \ linux-386 \ linux-amd64 \ linux-armv5 \ @@ -42,6 +41,7 @@ WINDOWS_ARCH_LIST = \ .PHONY: \ darwin-386 \ darwin-amd64 \ + darwin-arm64 \ linux-386 \ linux-amd64 \ linux-armv5 \ @@ -68,11 +68,7 @@ WINDOWS_ARCH_LIST = \ releases \ clean \ test \ - fmt \ - rpm \ - debian \ - debian-packages \ - docker-image + fmt darwin-386: GOARCH=386 GOOS=darwin $(GOBUILD) -o $(BINDIR)/$(NAME)-$@ $(GOFILES) diff --git a/cmd/archive.is/is.go b/cmd/archive.is/is.go new file mode 100644 index 0000000..c528992 --- /dev/null +++ b/cmd/archive.is/is.go @@ -0,0 +1,54 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/wabarc/archive.is" +) + +func main() { + var ( + playback bool + version bool + ) + + const playbackHelp = "Search archived URL" + const versionHelp = "Show version" + + flag.BoolVar(&playback, "playback", false, playbackHelp) + flag.BoolVar(&playback, "p", false, playbackHelp) + flag.BoolVar(&version, "version", false, versionHelp) + flag.BoolVar(&version, "v", false, versionHelp) + flag.Parse() + + if version { + fmt.Println(is.Version) + os.Exit(0) + } + + args := flag.Args() + if len(args) < 1 { + flag.Usage() + e := os.Args[0] + fmt.Printf(" %s url [url]\n\n", e) + fmt.Printf("example:\n %s https://example.com https://example.org\n\n", e) + os.Exit(1) + } + + wbrc := &is.Archiver{} + + if playback { + collects, _ := wbrc.Playback(args) + for orig, dest := range collects { + fmt.Println(orig, "=>", dest) + } + os.Exit(0) + } + + saved, _ := wbrc.Wayback(args) + for orig, dest := range saved { + fmt.Println(orig, "=>", dest) + } +} diff --git a/cmd/is.go b/cmd/is.go deleted file mode 100644 index 5b90c49..0000000 --- a/cmd/is.go +++ /dev/null @@ -1,28 +0,0 @@ -package is - -import ( - "flag" - "fmt" - "os" - - "github.com/wabarc/archive.is/pkg" -) - -func Run() { - flag.Parse() - - args := flag.Args() - if len(args) < 1 { - flag.Usage() - e := os.Args[0] - fmt.Printf(" %s url [url]\n\n", e) - fmt.Printf("example:\n %s https://www.google.com https://www.bbc.co.uk/\n\n", e) - os.Exit(1) - } - - wbrc := &is.Archiver{} - saved, _ := wbrc.Wayback(args) - for orig, dest := range saved { - fmt.Println(orig, "=>", dest) - } -} diff --git a/pkg/doc.go b/doc.go similarity index 100% rename from pkg/doc.go rename to doc.go diff --git a/go.mod b/go.mod index ebd6293..b81c08b 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,5 @@ require ( github.com/cretz/bine v0.1.0 github.com/stretchr/testify v1.7.0 // indirect github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616 + github.com/wabarc/logger v0.0.0-20210417045349-d0d82e8e99ee ) diff --git a/go.sum b/go.sum index 23da739..17354fe 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616 h1:wZ5HtpmZAVUq0Im5Sm92ycJrTeLJk5lB/Kvh55Rd+Ps= github.com/wabarc/helper v0.0.0-20210127120855-10af37cc2616/go.mod h1:N9P4r7Rn46p4nkWtXV6ztN3p5ACVnp++bgfwjTqSxQ8= +github.com/wabarc/logger v0.0.0-20210417045349-d0d82e8e99ee h1:MMIp++7eem2CI1jIYDoPByMwXeZAjsFo2ciBNtvhB80= +github.com/wabarc/logger v0.0.0-20210417045349-d0d82e8e99ee/go.mod h1:4uYr9fnQaQoDk1ttTzLnSB3lZm3i/vrJwN8EZIB2YuI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/is.go b/is.go new file mode 100644 index 0000000..87436cd --- /dev/null +++ b/is.go @@ -0,0 +1,336 @@ +package is + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/wabarc/helper" + "github.com/wabarc/logger" +) + +type Archiver struct { + Anyway string + Cookie string + + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + SkipTLSVerification bool +} + +type IS struct { + wbrc *Archiver + + submitid string + + endpoint *url.URL + httpClient *http.Client + torClient *http.Client +} + +var ( + userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36" + anyway = "0" + scheme = "http" + onion = "archivecaslytosk.onion" // archiveiya74codqgiixo33q62qlrqtkgmcitqx5u2oeqnmn5bpcbiyd.onion + cookie = "" + timeout = 120 * time.Second + domains = []string{ + "archive.today", + "archive.is", + "archive.li", + "archive.vn", + "archive.fo", + "archive.md", + "archive.ph", + } +) + +// Wayback is the handle of saving webpages to archive.is +func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) { + collects, results := make(map[string]string), make(map[string]string) + for _, link := range links { + if !helper.IsURL(link) { + logger.Info(link + " is invalid url.") + continue + } + collects[link] = link + } + + torClient, tor, err := newTorClient() + if err != nil { + logger.Error("%v", err) + } else { + defer tor.Close() + } + + ch := make(chan string, len(collects)) + defer close(ch) + + var mu sync.Mutex + var wg sync.WaitGroup + for _, link := range collects { + wg.Add(1) + go func(link string) { + is := &IS{ + wbrc: wbrc, + httpClient: &http.Client{Timeout: timeout, CheckRedirect: noRedirect}, + torClient: torClient, + submitid: "", + } + mu.Lock() + is.archive(link, ch) + results[link] = strings.Replace(<-ch, onion, "archive.today", 1) + mu.Unlock() + wg.Done() + }(link) + } + wg.Wait() + + if len(results) == 0 { + return results, fmt.Errorf("No results") + } + + return results, nil +} + +// Playback handle searching archived webpages from archive.is +func (wbrc *Archiver) Playback(links []string) (map[string]string, error) { + collects, results := make(map[string]string), make(map[string]string) + for _, link := range links { + if helper.IsURL(link) { + collects[link] = link + } + } + if len(collects) == 0 { + return results, fmt.Errorf("No found URL") + } + + torClient, tor, err := newTorClient() + if err != nil { + logger.Error("%v", err) + } else { + defer tor.Close() + } + + ch := make(chan string, len(collects)) + defer close(ch) + + var mu sync.Mutex + var wg sync.WaitGroup + for _, link := range collects { + wg.Add(1) + go func(link string) { + is := &IS{ + wbrc: wbrc, + httpClient: &http.Client{Timeout: timeout, CheckRedirect: noRedirect}, + torClient: torClient, + submitid: "", + } + mu.Lock() + is.search(link, ch) + results[link] = strings.Replace(<-ch, onion, "archive.today", 1) + mu.Unlock() + wg.Done() + }(link) + } + wg.Wait() + + if len(results) == 0 { + return results, fmt.Errorf("No results") + } + + return results, nil +} +func (is *IS) archive(uri string, ch chan<- string) { + _, err := is.getValidDomain() + if err != nil { + ch <- fmt.Sprint("archive.today is unavailable.") + return + } + + if is.wbrc.Anyway != "" { + anyway = is.wbrc.Anyway + } + data := url.Values{ + "submitid": {is.submitid}, + "anyway": {anyway}, + "url": {uri}, + } + domain := is.endpoint.String() + req, err := http.NewRequest("POST", domain+"/submit/", strings.NewReader(data.Encode())) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) + req.Header.Add("User-Agent", userAgent) + req.Header.Add("Referer", domain) + req.Header.Add("Origin", domain) + req.Header.Add("Host", is.endpoint.Hostname()) + req.Header.Add("Cookie", is.getCookie()) + resp, err := is.httpClient.Do(req) + if err != nil { + ch <- fmt.Sprint(err) + return + } + defer resp.Body.Close() + + code := resp.StatusCode / 100 + if code == 1 || code == 4 || code == 5 { + final := fmt.Sprintf("%s?url=%s", domain, uri) + ch <- final + return + } + + _, err = io.Copy(ioutil.Discard, resp.Body) + if err != nil { + ch <- fmt.Sprint(err) + return + } + + loc := resp.Header.Get("location") + if len(loc) > 2 { + ch <- loc + return + } + // Redirect to final url if page saved. + final := resp.Request.URL.String() + if len(final) > 0 && strings.Contains(final, "/submit/") == false { + ch <- final + return + } + // When use anyway parameter. + refresh := resp.Header.Get("refresh") + if len(refresh) > 0 { + r := strings.Split(refresh, ";url=") + if len(r) == 2 { + ch <- r[1] + return + } + } + + ch <- fmt.Sprintf("%s/timegate/%s", domain, uri) +} + +func noRedirect(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +func (is *IS) getCookie() string { + c := os.Getenv("ARCHIVE_COOKIE") + if c != "" { + is.wbrc.Cookie = c + } + + if is.wbrc.Cookie != "" { + return is.wbrc.Cookie + } else { + return cookie + } +} + +func (is *IS) getSubmitID(url string) (string, error) { + if strings.Contains(url, "http") == false { + return "", fmt.Errorf("missing protocol scheme") + } + + r := strings.NewReader("") + req, err := http.NewRequest("GET", url, r) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("User-Agent", userAgent) + req.Header.Add("Cookie", is.getCookie()) + resp, err := is.httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("status code error: %d", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return "", err + } + + id, exists := doc.Find("input[name=submitid]").First().Attr("value") + if !exists { + return "", fmt.Errorf("submitid not found") + } + + return id, nil +} + +func (is *IS) getValidDomain() (*url.URL, error) { + // get valid domain and submitid + r := func(domains []string) { + for _, domain := range domains { + h := fmt.Sprintf("%v://%v", scheme, domain) + id, err := is.getSubmitID(h) + if err != nil { + continue + } + is.endpoint, _ = url.Parse(h) + is.submitid = id + break + } + } + + // Try request over Tor hidden service. + if is.torClient != nil { + is.httpClient = is.torClient + + r([]string{onion}) + } + + if is.endpoint == nil || is.submitid == "" { + r(domains) + if is.endpoint == nil || is.submitid == "" { + return nil, fmt.Errorf("archive.today is unavailable.") + } + } + + return is.endpoint, nil +} + +func (is *IS) search(uri string, ch chan<- string) { + _, err := is.getValidDomain() + if err != nil { + ch <- fmt.Sprint("archive.today is unavailable.") + return + } + + domain := is.endpoint.String() + req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", domain, uri), nil) + req.Header.Add("User-Agent", userAgent) + req.Header.Add("Referer", domain) + req.Header.Add("Host", is.endpoint.Hostname()) + resp, err := is.httpClient.Do(req) + if err != nil { + ch <- fmt.Sprint(err) + return + } + defer resp.Body.Close() + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + ch <- fmt.Sprint(err) + return + } + + target, exists := doc.Find("#row0 > .TEXT-BLOCK > a").Attr("href") + if !exists { + ch <- "No found" + return + } + + ch <- target +} diff --git a/is_test.go b/is_test.go new file mode 100644 index 0000000..5da1ed0 --- /dev/null +++ b/is_test.go @@ -0,0 +1,87 @@ +package is + +import ( + "testing" +) + +func TestWayback(t *testing.T) { + var got map[string]string + + tests := []struct { + name string + urls []string + got int + }{ + { + name: "Without URLs", + urls: []string{}, + got: 0, + }, + { + name: "Has one invalid URL", + urls: []string{"foo bar", "https://example.com/"}, + got: 1, + }, + { + name: "URLs full matches", + urls: []string{"https://example.com/", "https://example.org/"}, + got: 2, + }, + } + + wbrc := &Archiver{} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _ = wbrc.Wayback(test.urls) + if len(got) != test.got { + t.Errorf("got = %d; want %d", len(got), test.got) + } + for orig, dest := range got { + if testing.Verbose() { + t.Log(orig, "=>", dest) + } + } + }) + } +} + +func TestPlayback(t *testing.T) { + var got map[string]string + + tests := []struct { + name string + urls []string + got int + }{ + { + name: "Without URLs", + urls: []string{}, + got: 0, + }, + { + name: "Has one invalid URL", + urls: []string{"foo bar", "https://example.com/"}, + got: 1, + }, + { + name: "URLs full matches", + urls: []string{"https://example.com/", "https://example.org/"}, + got: 2, + }, + } + + wbrc := &Archiver{} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _ = wbrc.Playback(test.urls) + if len(got) != test.got { + t.Errorf("got = %d; want %d", len(got), test.got) + } + for orig, dest := range got { + if testing.Verbose() { + t.Log(orig, "=>", dest) + } + } + }) + } +} diff --git a/main.go b/main.go deleted file mode 100644 index 4daebf8..0000000 --- a/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/wabarc/archive.is/cmd" - -func main() { - is.Run() -} diff --git a/main_test.go b/main_test.go deleted file mode 100644 index c7fbdde..0000000 --- a/main_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "testing" - - "github.com/wabarc/archive.is/pkg" -) - -func TestWayback(t *testing.T) { - links := []string{"https://www.google.com"} - wbrc := &is.Archiver{} - got, _ := wbrc.Wayback(links) - for _, dest := range got { - if dest == "" { - t.Error(got) - t.Fail() - } - } -} diff --git a/pkg/http.go b/pkg/http.go deleted file mode 100644 index 1713335..0000000 --- a/pkg/http.go +++ /dev/null @@ -1,178 +0,0 @@ -package is - -import ( - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/PuerkitoBio/goquery" -) - -type IS struct { - wbrc *Archiver - - submitid string - final string - - baseuri *url.URL - httpClient *http.Client - torClient *http.Client -} - -var ( - userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36" - anyway = "0" - scheme = "http" - onion = "archivecaslytosk.onion" // archiveiya74codqgiixo33q62qlrqtkgmcitqx5u2oeqnmn5bpcbiyd.onion - cookie = "" - timeout = 120 * time.Second - domains = []string{ - "archive.today", - "archive.is", - "archive.li", - "archive.vn", - "archive.fo", - "archive.md", - "archive.ph", - } -) - -func (is *IS) fetch(s string, ch chan<- string) { - // get valid domain and submitid - r := func(domains []string) { - for _, domain := range domains { - h := fmt.Sprintf("%v://%v", scheme, domain) - id, err := is.getSubmitID(h) - if err != nil { - continue - } - is.baseuri, _ = url.Parse(h) - is.submitid = id - break - } - } - - // Try request over Tor hidden service. - if is.torClient != nil { - is.httpClient = is.torClient - - r([]string{onion}) - } - - if is.baseuri == nil || is.submitid == "" { - r(domains) - if is.baseuri == nil || is.submitid == "" { - ch <- fmt.Sprint("archive.today is unavailable.") - return - } - } - - if is.wbrc.Anyway != "" { - anyway = is.wbrc.Anyway - } - data := url.Values{ - "submitid": {is.submitid}, - "anyway": {anyway}, - "url": {s}, - } - uri := is.baseuri.String() - req, err := http.NewRequest("POST", is.baseuri.String()+"/submit/", strings.NewReader(data.Encode())) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) - req.Header.Add("User-Agent", userAgent) - req.Header.Add("Referer", uri) - req.Header.Add("Origin", uri) - req.Header.Add("Host", is.baseuri.Hostname()) - req.Header.Add("Cookie", is.getCookie()) - resp, err := is.httpClient.Do(req) - if err != nil { - ch <- fmt.Sprint(err) - return - } - defer resp.Body.Close() - - code := resp.StatusCode / 100 - if code == 1 || code == 4 || code == 5 { - final := fmt.Sprintf("%s?url=%s", uri, s) - ch <- final - return - } - - _, err = io.Copy(ioutil.Discard, resp.Body) - if err != nil { - ch <- fmt.Sprint(err) - return - } - - // Redirect to final url if page saved. - final := resp.Request.URL.String() - if len(final) > 0 && strings.Contains(final, "/submit/") == false { - is.final = final - } - loc := resp.Header.Get("location") - if len(loc) > 2 { - is.final = loc - } - // When use anyway parameter. - refresh := resp.Header.Get("refresh") - if len(refresh) > 0 { - r := strings.Split(refresh, ";url=") - if len(r) == 2 { - is.final = r[1] - } - } - - ch <- is.final -} - -func (is *IS) getCookie() string { - c := os.Getenv("ARCHIVE_COOKIE") - if c != "" { - is.wbrc.Cookie = c - } - - if is.wbrc.Cookie != "" { - return is.wbrc.Cookie - } else { - return cookie - } -} - -func (is *IS) getSubmitID(url string) (string, error) { - if strings.Contains(url, "http") == false { - return "", fmt.Errorf("missing protocol scheme") - } - - r := strings.NewReader("") - req, err := http.NewRequest("GET", url, r) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Add("User-Agent", userAgent) - req.Header.Add("Cookie", is.getCookie()) - resp, err := is.httpClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("status code error: %d", resp.StatusCode) - } - - doc, err := goquery.NewDocumentFromReader(resp.Body) - if err != nil { - return "", err - } - - id, exists := doc.Find("input[name=submitid]").First().Attr("value") - if !exists { - return "", fmt.Errorf("submitid not found") - } - - return id, nil -} diff --git a/pkg/is.go b/pkg/is.go deleted file mode 100644 index dbd9646..0000000 --- a/pkg/is.go +++ /dev/null @@ -1,70 +0,0 @@ -package is - -import ( - "context" - "fmt" - "log" - "net" - "net/http" - "strings" - "sync" - - "github.com/wabarc/helper" -) - -type Archiver struct { - Anyway string - Cookie string - - DialContext func(ctx context.Context, network, addr string) (net.Conn, error) - SkipTLSVerification bool -} - -// Wayback is the handle of saving webpages to archive.is -func (wbrc *Archiver) Wayback(links []string) (map[string]string, error) { - collect, results := make(map[string]string), make(map[string]string) - for _, link := range links { - if !helper.IsURL(link) { - log.Print(link + " is invalid url.") - continue - } - collect[link] = link - } - - torClient, tor, err := newTorClient() - if err != nil { - log.Println(err) - } else { - defer tor.Close() - } - - ch := make(chan string, len(collect)) - defer close(ch) - - var mu sync.Mutex - var wg sync.WaitGroup - for link := range collect { - wg.Add(1) - go func(link string) { - is := &IS{ - wbrc: wbrc, - httpClient: &http.Client{Timeout: timeout}, - torClient: torClient, - final: "", - submitid: "", - } - is.fetch(link, ch) - mu.Lock() - results[link] = strings.Replace(<-ch, onion, "archive.today", 1) - mu.Unlock() - wg.Done() - }(link) - } - wg.Wait() - - if len(results) == 0 { - return results, fmt.Errorf("No results") - } - - return results, nil -} diff --git a/pkg/is_test.go b/pkg/is_test.go deleted file mode 100644 index 71a1608..0000000 --- a/pkg/is_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package is - -import ( - "testing" -) - -func TestWayback(t *testing.T) { - var ( - links []string - got map[string]string - ) - - wbrc := &Archiver{} - got, _ = wbrc.Wayback(links) - if len(got) != 0 { - t.Errorf("got = %d; want 0", len(got)) - } - - links = []string{"https://www.bbc.com/", "https://www.google.com/"} - got, _ = wbrc.Wayback(links) - if len(got) == 0 { - t.Errorf("got = %d; want greater than 0", len(got)) - } - - for orig, dest := range got { - t.Log(orig, "=>", dest) - } -} diff --git a/pkg/tor.go b/tor.go similarity index 96% rename from pkg/tor.go rename to tor.go index ec4155d..7d2df56 100644 --- a/pkg/tor.go +++ b/tor.go @@ -51,7 +51,8 @@ func newTorClient() (*http.Client, *tor.Tor, error) { // }) return &http.Client{ - Timeout: timeout, + Timeout: timeout, + CheckRedirect: noRedirect, Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: dialer.DialContext, diff --git a/version.go b/version.go index 2ff083a..1023b49 100644 --- a/version.go +++ b/version.go @@ -1,12 +1,3 @@ -package main +package is -import "fmt" - -var ( - version = "0.0.1" - date = "unknown" -) - -func init() { - fmt.Printf("version: %s\ndate: %s\n\n", version, date) -} +var Version = "dev"