mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-18 18:06:45 +00:00
48 lines
739 B
Go
48 lines
739 B
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net/url"
|
|
)
|
|
|
|
const (
|
|
maxTimeoutRetries = 3
|
|
)
|
|
|
|
type File struct {
|
|
Name string `json:"name"`
|
|
Size int64 `json:"size"`
|
|
Mtime int `json:"mtime"`
|
|
Path string `json:"path"`
|
|
IsDir bool `json:"-"`
|
|
}
|
|
|
|
type RemoteDir interface {
|
|
ListDir(path string)
|
|
}
|
|
|
|
func GetRemoteDir(u url.URL) (RemoteDir, error) {
|
|
switch u.Scheme {
|
|
case "http", "https":
|
|
return nil, nil //&HttpDirectory{}, nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported scheme: %s", u.Scheme)
|
|
}
|
|
}
|
|
|
|
type CrawlResult struct {
|
|
FileCount int
|
|
Status string
|
|
}
|
|
|
|
type RemoteDirCrawler struct {
|
|
Url string
|
|
MaxThreads int
|
|
// CrawledPaths
|
|
StatusCode string
|
|
}
|
|
|
|
func (r *RemoteDirCrawler) CrawlDir(outFile string) CrawlResult {
|
|
return CrawlResult{}
|
|
}
|