import csv
import json
from pygments import highlight
from pygments.lexers.data import JsonLexer
from pygments.formatters.html import HtmlFormatter
from ajax_select import make_ajax_form
from django.contrib import admin
from django.http import HttpResponse
from django.utils.translation import ugettext_lazy as _
from django.utils.safestring import mark_safe
from commonnet.admin_site import admin_site
from commcrawler import models
def export_as_csv_action(field_names,
description=_("Export selected as CSV file"),
header=True):
def export_as_csv(modeladmin, request, queryset):
"""
Generic csv export admin action.
based on http://djangosnippets.org/snippets/1697/
"""
opts = modeladmin.model._meta
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename=%s.csv' % \
str(opts).replace('.', '_')
writer = csv.writer(response)
if header:
writer.writerow(list(field_names))
for obj in queryset.values(*field_names).order_by('pk'):
row = []
for field in field_names:
value = obj[field]
if value is None:
value = ""
else:
value = str(value)
row.append(value)
writer.writerow(row)
return response
export_as_csv.short_description = description
return export_as_csv
class CrawlAdmin(admin.ModelAdmin):
model = models.Crawl
list_display = ("name", "status", "target_nb", "created", "started",
"pre_crawl_ended", "crawl_ended", "ended", "progress")
list_filter = ("status",)
exclude = ("progression", "created", "started", "pre_crawl_ended",
"crawl_ended", "ended")
readonly_fields = ()
form = make_ajax_form(model, {'targets': 'target'})
admin_site.register(models.Crawl, CrawlAdmin)
class CrawlResultAdmin(admin.ModelAdmin):
model = models.CrawlResult
list_display = (
"short_name", "open_link", "crawl", "started", "duration", "status",
"is_online", "bad_ssl", "nb_external_link", "nb_internal_link",
"nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube",
"nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf",
"nb_external_pdf", "nb_internal_office", "nb_external_office"
)
list_filter = ("status", "crawl", "is_online", "bad_ssl")
search_fields = ("target__name",)
readonly_fields = (
"started", "duration", "status", "nb_external_link", "nb_internal_link",
"nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube",
"nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf",
"nb_external_pdf", "nb_internal_office", "nb_external_office",
"is_online", "redirection", "crawl_result_prettified"
)
exclude = ("crawl_result",)
form = make_ajax_form(model, {'target': 'target'})
actions = [
export_as_csv_action(
("target_id", "target__name", "target__url", "nb_external_link",
"nb_internal_link", "nb_images", "nb_facebook", "nb_twitter",
"nb_instagram", "nb_youtube", "nb_dailymotion", "nb_vimeo",
"nb_video", "nb_audio", "nb_internal_pdf", "nb_external_pdf",
"nb_internal_office", "nb_external_office", "is_online",
"redirection")
)
]
def open_link(self, obj):
url = obj.url()
if not url:
return "-"
return mark_safe("{}".format(url, url))
def crawl_result_prettified(self, instance):
response = json.dumps(instance.crawl_result, sort_keys=True, indent=2)
formatter = HtmlFormatter(style='colorful')
response = highlight(response, JsonLexer(), formatter)
style = "
"
return mark_safe(style + response)
crawl_result_prettified.short_description = _("Crawl result")
admin_site.register(models.CrawlResult, CrawlResultAdmin)
class CrawlRelationAdmin(admin.ModelAdmin):
model = models.CrawlRelation
list_display = ("crawl", "source", "destination", "number")
list_filter = ("crawl",)
search_fields = ["source__name", "destination__name"]
form = make_ajax_form(model, {'source': 'target', 'destination': 'target'})
actions = [
export_as_csv_action(
("source_id", "destination_id", "number")
)
]
admin_site.register(models.CrawlRelation, CrawlRelationAdmin)
class ExcludedDomainAdmin(admin.ModelAdmin):
list_display = ('domain',)
search_fields = ('domain',)
admin_site.register(models.ExludedDomains, ExcludedDomainAdmin)