Basic scrapy configuration

1 files changed, 25 insertions, 0 deletions

diff --git a/commonnet/settings.py b/commonnet/settings.py
index 26c5c66..b51b83e 100644
--- a/commonnet/settings.py
+++ b/commonnet/settings.py
@@ -1,5 +1,8 @@
 import os
 
+from .scrapy_setting import SCRAPPY_SETTINGS
+from .version import VERSION
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 # SECURITY WARNING: keep the secret key used in production secret!
@@ -7,6 +10,8 @@ SECRET_KEY = '!lh+r$hzd(_-aj8a2&@)34bat=w&=!k+9w%$_+&^gjhf#n6z42'
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
+# Cache for http request
+DEV = True
 
 ALLOWED_HOSTS = []
 
@@ -106,3 +111,23 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = 5000
 # https://docs.djangoproject.com/en/1.11/howto/static-files/
 
 STATIC_URL = '/static/'
+
+RESPONSIBLE_EMAIL = None
+
+try:
+    from .local_settings import *
+except ImportError:
+    print("ERROR: a local_settings.py must be defined")
+    exit(1)
+
+if not RESPONSIBLE_EMAIL:
+    print("ERROR: a RESPONSIBLE_EMAIL must be defined in your "
+          "local_settings.py")
+    exit(1)
+
+SCRAPPY_SETTINGS["USER_AGENT"] = "commonnet-scraper v{} ({})".format(
+    VERSION, RESPONSIBLE_EMAIL
+)
+
+if DEV:
+    SCRAPPY_SETTINGS["HTTPCACHE_ENABLED"] = True