"""Renders admin pages for ops and other management tasks.
Currently just ``/admin/responses``, which shows active responses with tasks that
haven't completed yet.
"""
import datetime
import itertools
import logging
from flask import request
from google.cloud import ndb
from google.cloud.ndb.stats import KindStat, KindPropertyNamePropertyTypeStat
from webutil import logs
from webutil.util import json_dumps, json_loads
from flask_app import app
import models
import util
from util import render_template
# Import source class files so their metaclasses are initialized.
from models import BlogPost, Response, Source
import bluesky, flickr, github, instagram, mastodon, reddit, tumblr, twitter, wordpress_rest
logger = logging.getLogger(__name__)
NUM_ENTITIES = 10
# Result of this query in BigQuery:
# SELECT count(*) FROM `brid-gy.datastore.Response` WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_RESPONSES = 25313804
# Result of this query in BigQuery:
# SELECT SUM(ARRAY_LENGTH(sent) + ARRAY_LENGTH(unsent) + ARRAY_LENGTH(error) + ARRAY_LENGTH(failed) + ARRAY_LENGTH(skipped))
# FROM `brid-gy.datastore.Response`
# WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_LINKS = 4355678
# Result of this query in BigQuery:
# SELECT SUM(ARRAY_LENGTH(sent))
# FROM `brid-gy.datastore.Response`
# WHERE updated < timestamp('2021-11-01T00:00:00Z')
ARCHIVED_SENT_LINKS = 2060197
[docs]
@app.route('/admin/responses')
def responses():
"""Find the most recently attempted responses and blog posts with error URLs."""
entities = []
for cls in (Response,): # BlogPost
for e in cls.query().order(-cls.updated):
if (len(entities) >= NUM_ENTITIES or
e.updated < util.now() - datetime.timedelta(hours=1)):
break
elif (not e.error and not e.unsent) or e.status == 'complete':
continue
e.links = [util.pretty_link(u, new_tab=True) for u in e.error + e.failed]
if e.key.kind() == 'Response':
e.response = json_loads(e.response_json)
e.activities = [json_loads(a) for a in e.activities_json]
else:
e.response = {'content': '[BlogPost]'}
e.activities = [{'url': e.key.id()}]
entities.append(e)
return render_template('admin_responses.html', responses=entities, logs=logs)
[docs]
@app.route('/admin/sources')
def sources():
"""Find sources whose last poll errored out."""
CLASSES = (flickr.Flickr, github.GitHub, mastodon.Mastodon, reddit.Reddit,
bluesky.Bluesky)
queries = [cls.query(Source.status == 'enabled',
Source.poll_status == 'error',
Source.rate_limited.IN((False, None)),
Source.features == 'listen',
).fetch_async(NUM_ENTITIES)
for cls in CLASSES]
return render_template(
'admin_sources.html',
sources=itertools.chain(*[q.get_result() for q in queries]),
logs=logs,
)
@app.route('/admin/mark_complete', methods=['POST'])
def mark_complete():
entities = ndb.get_multi(ndb.Key(urlsafe=u)
for u in request.values.getlist('key'))
for e in entities:
e.status = 'complete'
ndb.put_multi(entities)
return util.redirect('/admin/responses')
@app.route('/admin/disable', methods=['POST'])
def disable():
source = util.load_source()
logger.info(f'Disabling {source.label()}')
source.status = 'disabled'
source.put()
return util.redirect(source.bridgy_path())
[docs]
@app.route('/admin/stats')
def stats():
"""Collect and report misc lifetime stats.
https://developers.google.com/appengine/docs/python/ndb/admin#Statistics_queries
Used to be on the front page, dropped them during the Flask port in August 2021.
"""
def count(query):
stat = query.get() # no datastore stats when running locally
return stat.count if stat else 0
def kind_count(kind):
return count(KindStat.query(KindStat.kind_name == kind))
num_users = sum(kind_count(cls.__name__) for cls in models.sources.values())
response_count = kind_count('Response')
link_counts = {
property: sum(count(KindPropertyNamePropertyTypeStat.query(
KindPropertyNamePropertyTypeStat.kind_name == kind,
KindPropertyNamePropertyTypeStat.property_name == property,
# specify string because there are also >2M Response entities with null
# values for some of these properties, as opposed to missing altogether,
# which we don't want to include.
KindPropertyNamePropertyTypeStat.property_type == 'String'))
for kind in ('BlogPost', 'Response'))
for property in ('sent', 'unsent', 'error', 'failed', 'skipped')}
return render_template('admin_stats.html', **{
# add comma separator between thousands
k: f'{v:,}' for k, v in {
'users': num_users,
'responses': response_count + ARCHIVED_RESPONSES,
'responses_stored': response_count,
'links': sum(link_counts.values()) + ARCHIVED_LINKS,
'webmentions': link_counts['sent'] + kind_count('BlogPost') + ARCHIVED_SENT_LINKS,
'publishes': kind_count('Publish'),
'blogposts': kind_count('BlogPost'),
'webmentions_received': kind_count('BlogWebmention'),
}.items()})