diff --git a/README.md b/README.md index 81459bc..4a6640f 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Usage: Available Commands: api host gophie as an API on a PORT env variable, fallback to set argument + clear-cache Clears the Gophie Cache engines Show summary and list of available engines help Help about any command list lists the recent movies by page number @@ -75,14 +76,15 @@ Available Commands: version Get Gophie Version Flags: - -e, --engine string The Engine to use for querying and downloading (default "netnaija") - -h, --help help for gophie - -o, --output-dir string Path to download files to - -v, --verbose Display Verbose logs + -c, --cache-dir string The directory to store/lookup cache + -e, --engine string The Engine to use for querying and downloading (default "netnaija") + -h, --help help for gophie + -o, --output-dir string Path to download files to + -s, --selenium-url string The URL of selenium instance to use + -v, --verbose Display Verbose logs Use "gophie [command] --help" for more information about a command. - Gophie - Bisoncorp (2020) (https://github.com/go-phie/gophie) ``` diff --git a/cmd/clearCache.go b/cmd/clear-cache.go similarity index 75% rename from cmd/clearCache.go rename to cmd/clear-cache.go index ab16150..704f058 100644 --- a/cmd/clearCache.go +++ b/cmd/clear-cache.go @@ -26,13 +26,7 @@ import ( // clearCacheCmd represents the clearCache command var clearCacheCmd = &cobra.Command{ Use: "clear-cache", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, + Short: "Clears the Gophie Cache", Run: func(cmd *cobra.Command, args []string) { err := os.RemoveAll(viper.GetString("cache-dir")) if err != nil { diff --git a/cmd/root.go b/cmd/root.go index 6c6c927..4fb15ef 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -43,6 +43,8 @@ var ( seleniumURL string // CacheDir: The Directory to store all colly files cacheDir string + // Should Cache requests or not + ignoreCache bool ) // rootCmd represents the base command when called without any subcommands @@ -86,6 +88,7 @@ func init() { &seleniumURL, "selenium-url", "s", "", "The URL of selenium instance to use") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Display Verbose logs") rootCmd.PersistentFlags().StringVarP(&outputPath, "output-dir", "o", "", "Path to download files to") + rootCmd.PersistentFlags().BoolVar(&ignoreCache, "ignore-cache", false, "Ignore Cache and makes new requests") viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config")) viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose")) @@ -93,6 +96,7 @@ func init() { viper.BindPFlag("selenium-url", rootCmd.PersistentFlags().Lookup("selenium-url")) viper.BindPFlag("output-dir", rootCmd.PersistentFlags().Lookup("output-dir")) viper.BindPFlag("cache-dir", rootCmd.PersistentFlags().Lookup("cache-dir")) + viper.BindPFlag("ignore-cache", rootCmd.PersistentFlags().Lookup("ignore-cache")) } @@ -114,10 +118,8 @@ func initConfig() { } // Configs From Env - replacer := strings.NewReplacer("-", "_") viper.SetEnvKeyReplacer(replacer) viper.SetEnvPrefix("gophie") // will be uppercased automatically viper.AutomaticEnv() // read in environment variables that match - } diff --git a/engine/engines.go b/engine/engines.go index 7ea6ee5..3c075c6 100644 --- a/engine/engines.go +++ b/engine/engines.go @@ -57,16 +57,23 @@ func Scrape(engine Engine) ([]Movie, error) { // Config Vars // seleniumURL := fmt.Sprintf("%s/wd/hub", viper.GetString("selenium-url")) cacheDir := viper.GetString("cache-dir") + ignoreCache := viper.GetBool("ignore-cache") var ( t *transport.ChromeDpTransport err error + c *colly.Collector ) - c := colly.NewCollector( - // Cache responses to prevent multiple download of pages - // even if the collector is restarted - colly.CacheDir(cacheDir), - ) + if ignoreCache { + c = colly.NewCollector() + + } else { + c = colly.NewCollector( + // Cache responses to prevent multiple download of pages + // even if the collector is restarted + colly.CacheDir(cacheDir), + ) + } // Add Cloud Flare scraper bypasser if engine.getName() == "NetNaija" {