1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
import csv
import json
from pygments import highlight
from pygments.lexers.data import JsonLexer
from pygments.formatters.html import HtmlFormatter
from ajax_select import make_ajax_form
from django.contrib import admin
from django.http import HttpResponse
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe
from commonnet.admin_site import admin_site
from commcrawler import models
def export_as_csv_action(field_names,
description=_("Export selected as CSV file"),
header=True):
def export_as_csv(modeladmin, request, queryset):
"""
Generic csv export admin action.
based on http://djangosnippets.org/snippets/1697/
"""
opts = modeladmin.model._meta
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=%s.csv' % \
str(opts).replace('.', '_')
writer = csv.writer(response)
if header:
writer.writerow(list(field_names))
for obj in queryset.values(*field_names).order_by('pk'):
row = []
for field in field_names:
value = obj[field]
if value is None:
value = ""
else:
value = str(value)
row.append(value)
writer.writerow(row)
return response
export_as_csv.short_description = description
return export_as_csv
class CrawlAdmin(admin.ModelAdmin):
model = models.Crawl
list_display = ("name", "status", "get_target_nb", "time_out", "created",
"started", "crawl_ended", "ended", "progress")
list_filter = ("status",)
exclude = ("progression", "created", "started", "pre_crawl_ended",
"crawl_ended", "ended")
readonly_fields = ()
form = make_ajax_form(model, {'targets': 'target'})
def get_target_nb(self, obj):
return obj.target_nb
get_target_nb.short_description = _("Target number")
admin_site.register(models.Crawl, CrawlAdmin)
class CrawlResultAdmin(admin.ModelAdmin):
model = models.CrawlResult
list_display = (
"short_name", "open_link", "crawl", "started", "duration", "status",
"is_online", "bad_ssl", "nb_external_link", "nb_internal_link",
"nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube",
"nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf",
"nb_external_pdf", "nb_internal_office", "nb_external_office"
)
list_filter = ("status", "crawl", "is_online", "bad_ssl")
search_fields = ("target__name",)
readonly_fields = (
"started", "duration", "status", "nb_external_link", "nb_internal_link",
"nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube",
"nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf",
"nb_external_pdf", "nb_internal_office", "nb_external_office",
"is_online", "redirection", "crawl_result_prettified"
)
exclude = ("crawl_result",)
form = make_ajax_form(model, {'target': 'target'})
actions = [
export_as_csv_action(
("target_id", "target__name", "target__url", "nb_external_link",
"nb_internal_link", "nb_images", "nb_facebook", "nb_twitter",
"nb_instagram", "nb_youtube", "nb_dailymotion", "nb_vimeo",
"nb_video", "nb_audio", "nb_internal_pdf", "nb_external_pdf",
"nb_internal_office", "nb_external_office", "is_online",
"redirection")
)
]
def open_link(self, obj):
url = obj.url()
if not url:
return "-"
return mark_safe("<a href='{}' target='blank_'>{}</a>".format(url, url))
def crawl_result_prettified(self, instance):
response = json.dumps(instance.crawl_result, sort_keys=True, indent=2)
formatter = HtmlFormatter(style='colorful')
response = highlight(response, JsonLexer(), formatter)
style = "<style>" + formatter.get_style_defs() + "</style><br>"
return mark_safe(style + response)
crawl_result_prettified.short_description = _("Crawl result")
admin_site.register(models.CrawlResult, CrawlResultAdmin)
class CrawlRelationAdmin(admin.ModelAdmin):
model = models.CrawlRelation
list_display = ("crawl", "source", "destination", "number")
list_filter = ("crawl",)
search_fields = ["source__name", "destination__name"]
form = make_ajax_form(model, {'source': 'target', 'destination': 'target'})
actions = [
export_as_csv_action(
("source_id", "destination_id", "number")
)
]
admin_site.register(models.CrawlRelation, CrawlRelationAdmin)
class ExcludedDomainAdmin(admin.ModelAdmin):
list_display = ('domain',)
search_fields = ('domain',)
admin_site.register(models.ExludedDomains, ExcludedDomainAdmin)
|
