1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2019-08-09 11:22
from __future__ import unicode_literals
import datetime
import django.contrib.postgres.fields.jsonb
import django.contrib.sites.models
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('commorganization', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Crawl',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=200, unique=True, verbose_name='Name')),
('created', models.DateTimeField(default=datetime.datetime.now, verbose_name='Creation')),
('started', models.DateTimeField(blank=True, null=True, verbose_name='Started')),
('crawl_ended', models.DateTimeField(blank=True, null=True, verbose_name='Crawl end')),
('ended', models.DateTimeField(blank=True, null=True, verbose_name='Ended')),
('status', models.CharField(choices=[('C', 'Created'), ('A', 'Planned'), ('P', 'Crawl in progress'), ('M', 'Match link in progress'), ('F', 'Finished')], default='C', max_length=1, verbose_name='Status')),
('progression', models.IntegerField(blank=True, null=True, verbose_name='Progression')),
('targets', models.ManyToManyField(blank=True, to='commorganization.Target')),
],
options={
'verbose_name_plural': 'Crawls',
'ordering': ('created', 'name'),
'verbose_name': 'Crawl',
},
),
migrations.CreateModel(
name='CrawlLink',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('link', models.URLField(verbose_name='Link')),
],
options={
'verbose_name_plural': 'Crawl links',
'verbose_name': 'Crawl link',
},
),
migrations.CreateModel(
name='CrawlRelation',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('number', models.IntegerField(default=1, verbose_name='Number')),
('crawl', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.Crawl', verbose_name='Crawl')),
('destination', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_destination', to='commorganization.Target', verbose_name='Destination')),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='relation_source', to='commorganization.Target', verbose_name='Source')),
],
options={
'verbose_name_plural': 'Crawl relations',
'verbose_name': 'Crawl relation',
},
),
migrations.CreateModel(
name='CrawlResult',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('started', models.DateTimeField(default=datetime.datetime.now, verbose_name='Start date')),
('duration', models.DurationField(blank=True, null=True, verbose_name='Duration')),
('status', models.CharField(choices=[('P', 'In progress'), ('T', 'Time out'), ('F', 'Finished')], default='P', max_length=1, verbose_name='Status')),
('crawl_result', django.contrib.postgres.fields.jsonb.JSONField(default=list, verbose_name='Crawl result')),
('nb_external_link', models.IntegerField(default=0, verbose_name='External links')),
('nb_internal_link', models.IntegerField(default=0, verbose_name='Internal links')),
('nb_images', models.IntegerField(default=0, verbose_name='Images')),
('nb_facebook', models.IntegerField(default=0, verbose_name='Facebook links')),
('nb_twitter', models.IntegerField(default=0, verbose_name='Twitter links')),
('nb_instagram', models.IntegerField(default=0, verbose_name='Instagram links')),
('nb_youtube', models.IntegerField(default=0, verbose_name='Youtube links')),
('nb_dailymotion', models.IntegerField(default=0, verbose_name='Dailymotion links')),
('nb_vimeo', models.IntegerField(default=0, verbose_name='Vimeo links')),
('nb_video', models.IntegerField(default=0, verbose_name='Internal videos')),
('nb_audio', models.IntegerField(default=0, verbose_name='Internal audios')),
('nb_internal_pdf', models.IntegerField(default=0, verbose_name='Internal PDF')),
('nb_external_pdf', models.IntegerField(default=0, verbose_name='External PDF')),
('nb_internal_office', models.IntegerField(default=0, verbose_name='Internal office documents')),
('nb_external_office', models.IntegerField(default=0, verbose_name='External office documents')),
('is_online', models.BooleanField(default=False, verbose_name='Website is online')),
('redirection', models.URLField(blank=True, null=True, verbose_name='Redirection')),
('crawl', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='results', to='commcrawler.Crawl', verbose_name='Crawl')),
('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commorganization.Target', verbose_name='Target')),
],
options={
'verbose_name_plural': 'Crawl results',
'verbose_name': 'Crawl result',
},
),
migrations.CreateModel(
name='ExludedDomains',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('domain', models.CharField(max_length=100, unique=True, validators=[django.contrib.sites.models._simple_domain_name_validator], verbose_name='Domain name')),
],
options={
'verbose_name_plural': 'Excluded domains',
'verbose_name': 'Excluded domain',
},
),
migrations.AddField(
model_name='crawllink',
name='result',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='commcrawler.CrawlResult', verbose_name='Result'),
),
migrations.AlterUniqueTogether(
name='crawlresult',
unique_together=set([('crawl', 'target')]),
),
migrations.AlterUniqueTogether(
name='crawlrelation',
unique_together=set([('crawl', 'source', 'destination')]),
),
]
|
