import csv import json from pygments import highlight from pygments.lexers.data import JsonLexer from pygments.formatters.html import HtmlFormatter from ajax_select import make_ajax_form from django.contrib import admin from django.http import HttpResponse from django.utils.translation import ugettext_lazy as _ from django.utils.safestring import mark_safe from commonnet.admin_site import admin_site from commcrawler import models def export_as_csv_action(field_names, description=_("Export selected as CSV file"), header=True): def export_as_csv(modeladmin, request, queryset): """ Generic csv export admin action. based on http://djangosnippets.org/snippets/1697/ """ opts = modeladmin.model._meta response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename=%s.csv' % \ str(opts).replace('.', '_') writer = csv.writer(response) if header: writer.writerow(list(field_names)) for obj in queryset.values(*field_names).order_by('pk'): row = [] for field in field_names: value = obj[field] if value is None: value = "" else: value = str(value) row.append(value) writer.writerow(row) return response export_as_csv.short_description = description return export_as_csv class CrawlAdmin(admin.ModelAdmin): model = models.Crawl list_display = ("name", "status", "target_nb", "created", "started", "pre_crawl_ended", "crawl_ended", "ended", "progress") list_filter = ("status",) exclude = ("progression", "created", "started", "pre_crawl_ended", "crawl_ended", "ended") readonly_fields = () form = make_ajax_form(model, {'targets': 'target'}) admin_site.register(models.Crawl, CrawlAdmin) class CrawlResultAdmin(admin.ModelAdmin): model = models.CrawlResult list_display = ( "short_name", "open_link", "crawl", "started", "duration", "status", "is_online", "bad_ssl", "nb_external_link", "nb_internal_link", "nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube", "nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf", "nb_external_pdf", "nb_internal_office", "nb_external_office" ) list_filter = ("status", "crawl", "is_online", "bad_ssl") search_fields = ("target__name",) readonly_fields = ( "started", "duration", "status", "nb_external_link", "nb_internal_link", "nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube", "nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf", "nb_external_pdf", "nb_internal_office", "nb_external_office", "is_online", "redirection", "crawl_result_prettified" ) exclude = ("crawl_result",) form = make_ajax_form(model, {'target': 'target'}) actions = [ export_as_csv_action( ("target_id", "target__name", "target__url", "nb_external_link", "nb_internal_link", "nb_images", "nb_facebook", "nb_twitter", "nb_instagram", "nb_youtube", "nb_dailymotion", "nb_vimeo", "nb_video", "nb_audio", "nb_internal_pdf", "nb_external_pdf", "nb_internal_office", "nb_external_office", "is_online", "redirection") ) ] def open_link(self, obj): url = obj.url() if not url: return "-" return mark_safe("{}".format(url, url)) def crawl_result_prettified(self, instance): response = json.dumps(instance.crawl_result, sort_keys=True, indent=2) formatter = HtmlFormatter(style='colorful') response = highlight(response, JsonLexer(), formatter) style = "
" return mark_safe(style + response) crawl_result_prettified.short_description = _("Crawl result") admin_site.register(models.CrawlResult, CrawlResultAdmin) class CrawlRelationAdmin(admin.ModelAdmin): model = models.CrawlRelation list_display = ("crawl", "source", "destination", "number") list_filter = ("crawl",) search_fields = ["source__name", "destination__name"] form = make_ajax_form(model, {'source': 'target', 'destination': 'target'}) actions = [ export_as_csv_action( ("source_id", "destination_id", "number") ) ] admin_site.register(models.CrawlRelation, CrawlRelationAdmin) class ExcludedDomainAdmin(admin.ModelAdmin): list_display = ('domain',) search_fields = ('domain',) admin_site.register(models.ExludedDomains, ExcludedDomainAdmin)