diff options
| author | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-07 03:53:32 +0200 |
|---|---|---|
| committer | Étienne Loks <etienne.loks@iggdrasil.net> | 2019-08-07 03:55:49 +0200 |
| commit | d737e04553f464966f54739ba37f9f06dab44586 (patch) | |
| tree | 2b68891ff1629b55b820312fdd3a17ce91ac5722 /commonnet | |
| parent | 9fbd94f70d4b819b45eef720425242c0d69b032d (diff) | |
| download | Comm-on-net-d737e04553f464966f54739ba37f9f06dab44586.tar.bz2 Comm-on-net-d737e04553f464966f54739ba37f9f06dab44586.zip | |
Save crawling results in the database
Diffstat (limited to 'commonnet')
| -rw-r--r-- | commonnet/local_settings.py.sample | 12 | ||||
| -rw-r--r-- | commonnet/scrapy_setting.py | 3 | ||||
| -rw-r--r-- | commonnet/settings.py | 13 |
3 files changed, 26 insertions, 2 deletions
diff --git a/commonnet/local_settings.py.sample b/commonnet/local_settings.py.sample index 0cba3f5..8c59516 100644 --- a/commonnet/local_settings.py.sample +++ b/commonnet/local_settings.py.sample @@ -4,3 +4,15 @@ RESPONSIBLE_EMAIL = None DEV = True EXTRA_APPS = [] + +# you probably have at least to set a password for db connection +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': 'commonnet', + 'USER': 'commonnet', + 'PASSWORD': '', + 'HOST': '127.0.0.1', + 'PORT': '5432', + } +} diff --git a/commonnet/scrapy_setting.py b/commonnet/scrapy_setting.py index 2d2b7b7..0d0f15b 100644 --- a/commonnet/scrapy_setting.py +++ b/commonnet/scrapy_setting.py @@ -85,5 +85,8 @@ SCRAPPY_SETTINGS = { #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' "COOKIES_ENABLED": False, + "ITEM_PIPELINES": { + 'commcrawler.scrapy.DbPipeline': 300, + } } diff --git a/commonnet/settings.py b/commonnet/settings.py index c72756c..40e3335 100644 --- a/commonnet/settings.py +++ b/commonnet/settings.py @@ -26,10 +26,13 @@ INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', + 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', ] +SITE_ID = 1 + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -64,14 +67,20 @@ WSGI_APPLICATION = 'commonnet.wsgi.application' # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases + DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'commonnet', + 'USER': 'commonnet', + 'PASSWORD': '', + 'HOST': '127.0.0.1', + 'PORT': '5432', } } + # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators |