From 07bac0728541458cb571771c142c23ae54ee4133 Mon Sep 17 00:00:00 2001 From: Hobbestigrou Date: Tue, 20 Feb 2018 16:20:34 +0100 Subject: [PATCH] [Core] Be pep8 compliant. --- experiments/admin.py | 51 ++-- experiments/admin_utils.py | 99 +++++-- experiments/apps.py | 10 +- experiments/conf.py | 9 +- experiments/counters.py | 31 ++- experiments/dateutils.py | 4 +- experiments/experiment_counters.py | 56 +++- experiments/manager.py | 14 +- experiments/middleware.py | 8 +- experiments/models.py | 25 +- experiments/signals.py | 3 +- experiments/significance.py | 17 +- experiments/stats.py | 8 +- experiments/templatetags/experiments.py | 31 ++- experiments/tests/test_admin.py | 89 ++++--- experiments/tests/test_signals.py | 9 +- experiments/tests/test_significance.py | 64 +++-- experiments/tests/test_templatetags.py | 28 +- experiments/tests/test_webuser.py | 225 +++++++++++----- experiments/tests/test_webuser_incorporate.py | 57 ++-- experiments/urls.py | 12 +- experiments/utils.py | 245 +++++++++++++----- experiments/views.py | 22 +- 23 files changed, 777 insertions(+), 340 deletions(-) diff --git a/experiments/admin.py b/experiments/admin.py index 3a9c469d..597148af 100644 --- a/experiments/admin.py +++ b/experiments/admin.py @@ -1,12 +1,15 @@ from django.contrib import admin from django.contrib.admin.utils import unquote +from django.conf.urls import url from django import forms -from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden +from django.http import ( + JsonResponse, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden) from django.utils import timezone + from experiments.admin_utils import get_result_context from experiments.models import Experiment from experiments import conf -from django.conf.urls import url + from experiments.utils import participant @@ -49,20 +52,25 @@ def get_form(self, request, obj=None, **kwargs): """ if obj: if obj.alternatives: - choices = [(alternative, alternative) for alternative in obj.alternatives.keys()] + choices = [ + (alternative, alternative) + for alternative in obj.alternatives.keys()] else: choices = [(conf.CONTROL_GROUP, conf.CONTROL_GROUP)] class ExperimentModelForm(forms.ModelForm): - default_alternative = forms.ChoiceField(choices=choices, - initial=obj.default_alternative, - required=False) + default_alternative = forms.ChoiceField( + choices=choices, initial=obj.default_alternative, + required=False) + kwargs['form'] = ExperimentModelForm - return super(ExperimentAdmin, self).get_form(request, obj=obj, **kwargs) + return super(ExperimentAdmin, self).get_form( + request, obj=obj, **kwargs) def save_model(self, request, obj, form, change): if change: - obj.set_default_alternative(form.cleaned_data['default_alternative']) + obj.set_default_alternative( + form.cleaned_data['default_alternative']) obj.save() # --------------------------------------- Overriding admin views @@ -90,22 +98,29 @@ def _admin_view_context(self, extra_context=None): return context def add_view(self, request, form_url='', extra_context=None): - return super(ExperimentAdmin, self).add_view(request, - form_url=form_url, - extra_context=self._admin_view_context(extra_context=extra_context)) + return super(ExperimentAdmin, self).add_view( + request, form_url=form_url, extra_context=self._admin_view_context( + extra_context=extra_context)) def change_view(self, request, object_id, form_url='', extra_context=None): experiment = self.get_object(request, unquote(object_id)) context = self._admin_view_context(extra_context=extra_context) context.update(get_result_context(request, experiment)) - return super(ExperimentAdmin, self).change_view(request, object_id, form_url=form_url, extra_context=context) + + return super(ExperimentAdmin, self).change_view( + request, object_id, form_url=form_url, extra_context=context) # --------------------------------------- Views for ajax functionality def get_urls(self): experiment_urls = [ - url(r'^set-alternative/$', self.admin_site.admin_view(self.set_alternative_view), name='experiment_admin_set_alternative'), - url(r'^set-state/$', self.admin_site.admin_view(self.set_state_view), name='experiment_admin_set_state'), + url( + r'^set-alternative/$', self.admin_site.admin_view( + self.set_alternative_view), + name='experiment_admin_set_alternative'), + url( + r'^set-state/$', self.admin_site.admin_view( + self.set_state_view), name='experiment_admin_set_state'), ] return experiment_urls + super(ExperimentAdmin, self).get_urls() @@ -124,7 +139,8 @@ def set_alternative_view(self, request): participant(request).set_alternative(experiment_name, alternative_name) return JsonResponse({ 'success': True, - 'alternative': participant(request).get_alternative(experiment_name) + 'alternative': participant( + request).get_alternative(experiment_name) }) def set_state_view(self, request): @@ -140,7 +156,8 @@ def set_state_view(self, request): return HttpResponseBadRequest() try: - experiment = Experiment.objects.get(name=request.POST.get("experiment")) + experiment = Experiment.objects.get( + name=request.POST.get("experiment")) except Experiment.DoesNotExist: return HttpResponseBadRequest() @@ -155,5 +172,5 @@ def set_state_view(self, request): return HttpResponse() -admin.site.register(Experiment, ExperimentAdmin) +admin.site.register(Experiment, ExperimentAdmin) diff --git a/experiments/admin_utils.py b/experiments/admin_utils.py index 6f405faf..245e4701 100644 --- a/experiments/admin_utils.py +++ b/experiments/admin_utils.py @@ -1,16 +1,16 @@ +import json + from experiments.experiment_counters import ExperimentCounter from experiments.significance import chi_square_p_value, mann_whitney from experiments.utils import participant from experiments import conf -import json - MIN_ACTIONS_TO_SHOW = 3 def rate(a, b): - if not b or a == None: + if not b or a is None: return None return 100. * a / b @@ -35,9 +35,11 @@ def chi_squared_confidence(a_count, a_conversion, b_count, b_conversion): def average_actions(distribution): total_users = 0 total_actions = 0 + for actions, frequency in distribution.items(): total_users += frequency total_actions += actions * frequency + if total_users: return total_actions / float(total_users) else: @@ -52,6 +54,7 @@ def fixup_distribution(distribution, count): def mann_whitney_confidence(a_distribution, b_distribution): p_value = mann_whitney(a_distribution, b_distribution)[1] + if p_value is not None: return (1 - p_value * 2) * 100 # Two tailed probability else: @@ -60,15 +63,17 @@ def mann_whitney_confidence(a_distribution, b_distribution): def points_with_surrounding_gaps(points): """ - This function makes sure that any gaps in the sequence provided have stopper points at their beginning - and end so a graph will be drawn with correct 0 ranges. This is more efficient than filling in all points - up to the maximum value. For example: + This function makes sure that any gaps in the sequence provided have + stopper points at their beginning and end so a graph will be drawn with + correct 0 ranges. This is more efficient than filling in all points up to + the maximum value. For example: input: [1,2,3,10,11,13] output [1,2,3,4,9,10,11,12,13] """ points_with_gaps = [] last_point = -1 + for point in points: if last_point + 1 == point: pass @@ -77,31 +82,46 @@ def points_with_surrounding_gaps(points): else: points_with_gaps.append(last_point + 1) points_with_gaps.append(point - 1) + points_with_gaps.append(point) last_point = point + return points_with_gaps def conversion_distributions_to_graph_table(conversion_distributions): ordered_distributions = list(conversion_distributions.items()) - total_entries = dict((name, float(sum(dist.values()) or 1)) for name, dist in ordered_distributions) + total_entries = dict(( + name, float(sum(dist.values()) or 1)) + for name, dist in ordered_distributions) graph_head = [['x'] + [name for name, dist in ordered_distributions]] - points_in_any_distribution = sorted(set(k for name, dist in ordered_distributions for k in dist.keys())) + points_in_any_distribution = sorted( + set(k for name, dist in ordered_distributions for k in dist.keys())) points_with_gaps = points_with_surrounding_gaps(points_in_any_distribution) - graph_body = [[point] + [dist.get(point, 0) / total_entries[name] for name, dist in ordered_distributions] for point in points_with_gaps] + graph_body = [ + [point] + [dist.get(point, 0) / total_entries[ + name] for name, dist in ordered_distributions] + for point in points_with_gaps] accumulator = [0] * len(ordered_distributions) for point in range(len(graph_body) - 1, -1, -1): - accumulator = [graph_body[point][j + 1] + accumulator[j] for j in range(len(ordered_distributions))] + accumulator = [ + graph_body[point][j + 1] + accumulator[j] for j in range(len( + ordered_distributions))] graph_body[point][1:] = accumulator - interesting_points = [point for point in points_in_any_distribution if max(dist.get(point, 0) for name, dist in ordered_distributions) >= MIN_ACTIONS_TO_SHOW] + interesting_points = [ + point for point in points_in_any_distribution if max( + dist.get(point, 0) + for name, dist in ordered_distributions) >= MIN_ACTIONS_TO_SHOW] if len(interesting_points): highest_interesting_point = max(interesting_points) else: highest_interesting_point = 0 - graph_body = [g for g in graph_body if g[0] <= highest_interesting_point and g[0] != 0] + graph_body = [ + g for g in graph_body if g[0] <= highest_interesting_point + and g[0] != 0] graph_table = graph_head + graph_body return json.dumps(graph_table) @@ -121,11 +141,15 @@ def get_result_context(request, experiment): relevant_goals = set(chi2_goals + mwu_goals) alternatives = {} + for alternative_name in experiment.alternatives.keys(): - alternatives[alternative_name] = experiment_counter.participant_count(experiment, alternative_name) + alternatives[alternative_name] = experiment_counter.participant_count( + experiment, alternative_name) + alternatives = sorted(alternatives.items()) - control_participants = experiment_counter.participant_count(experiment, conf.CONTROL_GROUP) + control_participants = experiment_counter.participant_count( + experiment, conf.CONTROL_GROUP) results = {} @@ -133,34 +157,53 @@ def get_result_context(request, experiment): show_mwu = goal in mwu_goals alternatives_conversions = {} - control_conversions = experiment_counter.goal_count(experiment, conf.CONTROL_GROUP, goal) - control_conversion_rate = rate(control_conversions, control_participants) + control_conversions = experiment_counter.goal_count( + experiment, conf.CONTROL_GROUP, goal) + control_conversion_rate = rate( + control_conversions, control_participants) if show_mwu: mwu_histogram = {} - control_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, conf.CONTROL_GROUP, goal), control_participants) - control_average_goal_actions = average_actions(control_conversion_distribution) + control_conversion_distribution = fixup_distribution( + experiment_counter.goal_distribution( + experiment, conf.CONTROL_GROUP, goal), + control_participants) + control_average_goal_actions = average_actions( + control_conversion_distribution) mwu_histogram['control'] = control_conversion_distribution else: control_average_goal_actions = None + for alternative_name in experiment.alternatives.keys(): if not alternative_name == conf.CONTROL_GROUP: - alternative_conversions = experiment_counter.goal_count(experiment, alternative_name, goal) - alternative_participants = experiment_counter.participant_count(experiment, alternative_name) - alternative_conversion_rate = rate(alternative_conversions, alternative_participants) - alternative_confidence = chi_squared_confidence(alternative_participants, alternative_conversions, control_participants, control_conversions) + alternative_conversions = experiment_counter.goal_count( + experiment, alternative_name, goal) + alternative_participants = experiment_counter\ + .participant_count(experiment, alternative_name) + alternative_conversion_rate = rate( + alternative_conversions, alternative_participants) + alternative_confidence = chi_squared_confidence( + alternative_participants, alternative_conversions, + control_participants, control_conversions) + if show_mwu: - alternative_conversion_distribution = fixup_distribution(experiment_counter.goal_distribution(experiment, alternative_name, goal), alternative_participants) - alternative_average_goal_actions = average_actions(alternative_conversion_distribution) + alternative_conversion_distribution = fixup_distribution( + experiment_counter.goal_distribution( + experiment, alternative_name, goal), + alternative_participants) + alternative_average_goal_actions = average_actions( + alternative_conversion_distribution) alternative_distribution_confidence = mann_whitney_confidence(alternative_conversion_distribution, control_conversion_distribution) mwu_histogram[alternative_name] = alternative_conversion_distribution else: alternative_average_goal_actions = None alternative_distribution_confidence = None + alternative = { 'conversions': alternative_conversions, 'conversion_rate': alternative_conversion_rate, - 'improvement': improvement(alternative_conversion_rate, control_conversion_rate), + 'improvement': improvement( + alternative_conversion_rate, control_conversion_rate), 'confidence': alternative_confidence, 'average_goal_actions': alternative_average_goal_actions, 'mann_whitney_confidence': alternative_distribution_confidence, @@ -178,7 +221,8 @@ def get_result_context(request, experiment): "alternatives": sorted(alternatives_conversions.items()), "relevant": goal in relevant_goals or relevant_goals == {u''}, "mwu": goal in mwu_goals, - "mwu_histogram": conversion_distributions_to_graph_table(mwu_histogram) if show_mwu else None + "mwu_histogram": conversion_distributions_to_graph_table( + mwu_histogram) if show_mwu else None } return { @@ -187,5 +231,6 @@ def get_result_context(request, experiment): 'control_participants': control_participants, 'results': results, 'column_count': len(alternatives_conversions) * 3 + 2, # Horrible coupling with template design - 'user_alternative': participant(request).get_alternative(experiment.name), + 'user_alternative': participant( + request).get_alternative(experiment.name), } diff --git a/experiments/apps.py b/experiments/apps.py index e7515bb0..f7a208fe 100644 --- a/experiments/apps.py +++ b/experiments/apps.py @@ -7,7 +7,11 @@ class ExperimentsConfig(AppConfig): def ready(self): from django.contrib.auth.signals import user_logged_in, user_logged_out - from experiments.signal_handlers import transfer_enrollments_to_user, handle_user_logged_out + from experiments.signal_handlers import ( + transfer_enrollments_to_user, handle_user_logged_out) - user_logged_in.connect(transfer_enrollments_to_user, dispatch_uid="experiments_user_logged_in") - user_logged_out.connect(handle_user_logged_out, dispatch_uid="experiments_user_logged_out") + user_logged_in.connect( + transfer_enrollments_to_user, + dispatch_uid="experiments_user_logged_in") + user_logged_out.connect( + handle_user_logged_out, dispatch_uid="experiments_user_logged_out") diff --git a/experiments/conf.py b/experiments/conf.py index 19c15d02..e36cf2d2 100644 --- a/experiments/conf.py +++ b/experiments/conf.py @@ -1,6 +1,7 @@ -from django.conf import settings -from itertools import chain import re +from itertools import chain + +from django.conf import settings CONTROL_GROUP = 'control' @@ -21,6 +22,8 @@ CONFIRM_HUMAN = getattr(settings, 'EXPERIMENTS_CONFIRM_HUMAN', True) -CONFIRM_HUMAN_SESSION_KEY = getattr(settings, 'EXPERIMENTS_CONFIRM_HUMAN_SESSION_KEY', 'experiments_verified_human') +CONFIRM_HUMAN_SESSION_KEY = getattr( + settings, 'EXPERIMENTS_CONFIRM_HUMAN_SESSION_KEY', + 'experiments_verified_human') BOT_REGEX = re.compile("(Baidu|Gigabot|Googlebot|YandexBot|AhrefsBot|TVersity|libwww-perl|Yeti|lwp-trivial|msnbot|bingbot|facebookexternalhit|Twitterbot|Twitmunin|SiteUptime|TwitterFeed|Slurp|WordPress|ZIBB|ZyBorg)", re.IGNORECASE) diff --git a/experiments/counters.py b/experiments/counters.py index d0429d22..fde154a7 100644 --- a/experiments/counters.py +++ b/experiments/counters.py @@ -15,8 +15,11 @@ class Counters(object): @cached_property def _redis(self): if getattr(settings, 'EXPERIMENTS_REDIS_SENTINELS', None): - sentinel = Sentinel(settings.EXPERIMENTS_REDIS_SENTINELS, socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT) - host, port = sentinel.discover_master(settings.EXPERIMENTS_REDIS_MASTER_NAME) + sentinel = Sentinel( + settings.EXPERIMENTS_REDIS_SENTINELS, + socket_timeout=settings.EXPERIMENTS_REDIS_SENTINELS_TIMEOUT) + host, port = sentinel.discover_master( + settings.EXPERIMENTS_REDIS_MASTER_NAME) else: host = getattr(settings, 'EXPERIMENTS_REDIS_HOST', 'localhost') port = getattr(settings, 'EXPERIMENTS_REDIS_PORT', 6379) @@ -33,7 +36,8 @@ def increment(self, key, participant_identifier, count=1): try: cache_key = COUNTER_CACHE_KEY % key freq_cache_key = COUNTER_FREQ_CACHE_KEY % key - new_value = self._redis.hincrby(cache_key, participant_identifier, count) + new_value = self._redis.hincrby( + cache_key, participant_identifier, count) # Maintain histogram of per-user counts if new_value > count: @@ -48,7 +52,9 @@ def clear(self, key, participant_identifier): # Remove the direct entry cache_key = COUNTER_CACHE_KEY % key pipe = self._redis.pipeline() - freq, _ = pipe.hget(cache_key, participant_identifier).hdel(cache_key, participant_identifier).execute() + freq, _ = pipe.hget( + cache_key, participant_identifier).hdel( + cache_key, participant_identifier).execute() # Remove from the histogram freq_cache_key = COUNTER_FREQ_CACHE_KEY % key @@ -77,11 +83,13 @@ def get_frequency(self, key, participant_identifier): def get_frequencies(self, key): try: freq_cache_key = COUNTER_FREQ_CACHE_KEY % key - # In some cases when there are concurrent updates going on, there can - # briefly be a negative result for some frequency count. We discard these - # as they shouldn't really affect the result, and they are about to become - # zero anyway. - return dict((int(k), int(v)) for (k, v) in self._redis.hgetall(freq_cache_key).items() if int(v) > 0) + # In some cases when there are concurrent updates going on, there + # can briefly be a negative result for some frequency count. We + # discard these as they shouldn't really affect the result, and + # they are about to become zero anyway. + return dict(( + int(k), int(v)) for (k, v) in self._redis.hgetall( + freq_cache_key).items() if int(v) > 0) except (ConnectionError, ResponseError): # Handle Redis failures gracefully return tuple() @@ -98,12 +106,15 @@ def reset(self, key): return False def reset_pattern(self, pattern_key): - #similar to above, but can pass pattern as arg instead + # similar to above, but can pass pattern as arg instead try: cache_key = COUNTER_CACHE_KEY % pattern_key + for key in self._redis.keys(cache_key): self._redis.delete(key) + freq_cache_key = COUNTER_FREQ_CACHE_KEY % pattern_key + for key in self._redis.keys(freq_cache_key): self._redis.delete(key) return True diff --git a/experiments/dateutils.py b/experiments/dateutils.py index e1c81de9..649197e0 100644 --- a/experiments/dateutils.py +++ b/experiments/dateutils.py @@ -11,7 +11,9 @@ def fix_awareness(value): - tz_aware_value = value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None + tz_aware_value = value.tzinfo is not None and value.tzinfo.utcoffset( + value) is not None + if USE_TZ and not tz_aware_value: from django.utils.timezone import get_current_timezone return value.replace(tzinfo=get_current_timezone()) diff --git a/experiments/experiment_counters.py b/experiments/experiment_counters.py index 820a953b..0b1ad10e 100644 --- a/experiments/experiment_counters.py +++ b/experiments/experiment_counters.py @@ -1,48 +1,78 @@ -from experiments import counters, conf import logging import json +from experiments import counters, conf + PARTICIPANT_KEY = '%s:%s:participant' GOAL_KEY = '%s:%s:%s:goal' logger = logging.getLogger('experiments') + class ExperimentCounter(object): def __init__(self): self.counters = counters.Counters() - def increment_participant_count(self, experiment, alternative_name, participant_identifier): + def increment_participant_count( + self, experiment, alternative_name, participant_identifier): counter_key = PARTICIPANT_KEY % (experiment.name, alternative_name) + self.counters.increment(counter_key, participant_identifier) - logger.info(json.dumps({'type':'participant_add', 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) + logger.info(json.dumps({ + 'type': 'participant_add', + 'experiment': experiment.name, + 'alternative': alternative_name, + 'participant': participant_identifier})) - def increment_goal_count(self, experiment, alternative_name, goal_name, participant_identifier, count=1): + def increment_goal_count( + self, experiment, alternative_name, goal_name, + participant_identifier, count=1): counter_key = GOAL_KEY % (experiment.name, alternative_name, goal_name) + self.counters.increment(counter_key, participant_identifier, count) - logger.info(json.dumps({'type':'goal_hit', 'goal': goal_name, 'goal_count': count, 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) + logger.info(json.dumps({ + 'type': 'goal_hit', + 'goal': goal_name, + 'goal_count': count, + 'experiment': experiment.name, + 'alternative': alternative_name, + 'participant': participant_identifier})) - def remove_participant(self, experiment, alternative_name, participant_identifier): + def remove_participant( + self, experiment, alternative_name, participant_identifier): counter_key = PARTICIPANT_KEY % (experiment.name, alternative_name) + self.counters.clear(counter_key, participant_identifier) - logger.info(json.dumps({'type':'participant_remove', 'experiment': experiment.name, 'alternative': alternative_name, 'participant': participant_identifier})) + logger.info(json.dumps({ + 'type': 'participant_remove', + 'experiment': experiment.name, + 'alternative': alternative_name, + 'participant': participant_identifier})) # Remove goal records for goal_name in conf.ALL_GOALS: - counter_key = GOAL_KEY % (experiment.name, alternative_name, goal_name) + counter_key = GOAL_KEY % ( + experiment.name, alternative_name, goal_name) self.counters.clear(counter_key, participant_identifier) def participant_count(self, experiment, alternative): - return self.counters.get(PARTICIPANT_KEY % (experiment.name, alternative)) + return self.counters.get( + PARTICIPANT_KEY % (experiment.name, alternative)) def goal_count(self, experiment, alternative, goal): - return self.counters.get(GOAL_KEY % (experiment.name, alternative, goal)) + return self.counters.get( + GOAL_KEY % (experiment.name, alternative, goal)) - def participant_goal_frequencies(self, experiment, alternative, participant_identifier): + def participant_goal_frequencies( + self, experiment, alternative, participant_identifier): for goal in conf.ALL_GOALS: - yield goal, self.counters.get_frequency(GOAL_KEY % (experiment.name, alternative, goal), participant_identifier) + yield goal, self.counters.get_frequency( + GOAL_KEY % (experiment.name, alternative, goal), + participant_identifier) def goal_distribution(self, experiment, alternative, goal): - return self.counters.get_frequencies(GOAL_KEY % (experiment.name, alternative, goal)) + return self.counters.get_frequencies( + GOAL_KEY % (experiment.name, alternative, goal)) def delete(self, experiment): self.counters.reset_pattern(experiment.name + "*") diff --git a/experiments/manager.py b/experiments/manager.py index f0807d69..f785299f 100644 --- a/experiments/manager.py +++ b/experiments/manager.py @@ -1,11 +1,14 @@ from django.conf import settings -from experiments.models import Experiment + from modeldict import ModelDict +from experiments.models import Experiment + class LazyAutoCreate(object): """ - A lazy version of the setting is used so that tests can change the setting and still work + A lazy version of the setting is used so that tests can change the setting + and still work """ def __nonzero__(self): return self.__bool__() @@ -16,11 +19,14 @@ def __bool__(self): class ExperimentManager(ModelDict): def get_experiment(self, experiment_name): - # Helper that uses self[...] so that the experiment is auto created where desired + # Helper that uses self[...] so that the experiment is auto created + # where desired try: return self[experiment_name] except KeyError: return None -experiment_manager = ExperimentManager(Experiment, key='name', value='value', instances=True, auto_create=LazyAutoCreate()) +experiment_manager = ExperimentManager( + Experiment, key='name', value='value', instances=True, + auto_create=LazyAutoCreate()) diff --git a/experiments/middleware.py b/experiments/middleware.py index 203b6e49..a6f34b9a 100644 --- a/experiments/middleware.py +++ b/experiments/middleware.py @@ -10,9 +10,11 @@ class ExperimentsRetentionMiddleware(MiddlewareMixin): def process_response(self, request, response): - # Don't track, failed pages, ajax requests, logged out users or widget impressions. - # We detect widgets by relying on the fact that they are flagged as being embedable - if response.status_code != 200 or request.is_ajax() or getattr(response, 'xframe_options_exempt', False): + # Don't track, failed pages, ajax requests, logged out users or widget + # impressions. We detect widgets by relying on the fact that they are + # flagged as being embedable + if response.status_code != 200 or request.is_ajax() or getattr( + response, 'xframe_options_exempt', False): return response experiment_user = participant(request) diff --git a/experiments/models.py b/experiments/models.py index edf0f4b0..1144bd98 100644 --- a/experiments/models.py +++ b/experiments/models.py @@ -1,12 +1,12 @@ +import random +import json + from django.db import models from django.core.serializers.json import DjangoJSONEncoder from django.conf import settings from jsonfield import JSONField -import random -import json - from experiments.dateutils import now from experiments import conf @@ -31,7 +31,8 @@ class Experiment(models.Model): state = models.IntegerField(default=CONTROL_STATE, choices=STATES) - start_date = models.DateTimeField(default=now, blank=True, null=True, db_index=True) + start_date = models.DateTimeField( + default=now, blank=True, null=True, db_index=True) end_date = models.DateTimeField(blank=True, null=True) def is_displaying_alternatives(self): @@ -59,7 +60,9 @@ def ensure_alternative_exists(self, alternative, weight=None): self.alternatives[alternative] = {} self.alternatives[alternative]['enabled'] = True self.save() - if weight is not None and 'weight' not in self.alternatives[alternative]: + + if weight is not None and 'weight' not in self.alternatives[ + alternative]: self.alternatives[alternative]['weight'] = float(weight) self.save() @@ -79,7 +82,9 @@ def set_default_alternative(self, alternative): def random_alternative(self): if all('weight' in alt for alt in self.alternatives.values()): - return weighted_choice([(name, details['weight']) for name, details in self.alternatives.items()]) + return weighted_choice( + [(name, details['weight']) + for name, details in self.alternatives.items()]) else: return random.choice(list(self.alternatives)) @@ -106,7 +111,9 @@ def to_dict_serialized(self): class Enrollment(models.Model): """ A participant in a split testing experiment """ - user = models.ForeignKey(getattr(settings, 'AUTH_USER_MODEL', 'auth.User'), on_delete=models.CASCADE) + user = models.ForeignKey( + getattr(settings, 'AUTH_USER_MODEL', 'auth.User'), + on_delete=models.CASCADE) experiment = models.ForeignKey(Experiment, on_delete=models.CASCADE) enrollment_date = models.DateTimeField(auto_now_add=True) last_seen = models.DateTimeField(null=True) @@ -123,9 +130,9 @@ def weighted_choice(choices): total = sum(w for c, w in choices) r = random.uniform(0, total) upto = 0 + for c, w in choices: upto += w + if upto >= r: return c - - diff --git a/experiments/signals.py b/experiments/signals.py index b26075a6..a430f1fb 100644 --- a/experiments/signals.py +++ b/experiments/signals.py @@ -1,3 +1,4 @@ from django.dispatch import Signal -user_enrolled = Signal(providing_args=['experiment', 'alternative', 'user', 'session']) +user_enrolled = Signal( + providing_args=['experiment', 'alternative', 'user', 'session']) diff --git a/experiments/significance.py b/experiments/significance.py index d2a265ed..50bd6bee 100644 --- a/experiments/significance.py +++ b/experiments/significance.py @@ -4,15 +4,12 @@ def mann_whitney(a_distribution, b_distribution, use_continuity=True): """Returns (u, p_value)""" MINIMUM_VALUES = 20 - all_values = sorted(set(a_distribution) | set(b_distribution)) - count_so_far = 0 a_rank_sum = 0 b_rank_sum = 0 a_count = 0 b_count = 0 - variance_adjustment = 0 for v in all_values: @@ -60,7 +57,8 @@ def mann_whitney(a_distribution, b_distribution, use_continuity=True): def chi_square_p_value(matrix): """ - Accepts a matrix (an array of arrays, where each child array represents a row) + Accepts a matrix (an array of arrays, where each child array represents a + row) Example from http://math.hws.edu/javamath/ryan/ChiSquare.html: @@ -122,16 +120,19 @@ def chi_square_p_value(matrix): observed_test_statistic = 0.0 for i in range(num_rows): for j in range(num_columns): - expected_value = (row_sums[i] / grand_total) * (column_sums[j] / grand_total) * grand_total + expected_value = ( + row_sums[i] / grand_total) * ( + column_sums[j] / grand_total) * grand_total + if expected_value <= 0: return None, None observed_value = matrix[i][j] - observed_test_statistic += ((observed_value - expected_value) ** 2) / expected_value + observed_test_statistic += (( + observed_value - expected_value) ** 2) / expected_value # See https://bitbucket.org/akoha/django-lean/issue/16/g_test-formula-is-incorrect - #observed_test_statistic += 2 * (observed_value*log(observed_value/expected_value)) + # observed_test_statistic += 2 * (observed_value*log(observed_value/expected_value)) degrees_freedom = (num_columns - 1) * (num_rows - 1) - p_value = chisqprob(observed_test_statistic, degrees_freedom) return observed_test_statistic, p_value diff --git a/experiments/stats.py b/experiments/stats.py index 70c26555..c8f5d6e8 100644 --- a/experiments/stats.py +++ b/experiments/stats.py @@ -4,7 +4,7 @@ def zprob(z): """ Returns the area under the normal curve 'to the left of' the given z value. - Thus, + Thus, for z<0, zprob(z) = 1-tail probability for z>0, 1.0-zprob(z) = 1-tail probability for any z, 2.0*(1.0-zprob(abs(z))) = 2-tail probability @@ -46,10 +46,10 @@ def zprob(z): def chisqprob(chisq, df): """ Returns the (1-tailed) probability value associated with the provided - chi-square value and df. - + chi-square value and df. + Originally adapted from Gary Perlman code by Gary Strangman. - + Usage: chisqprob(chisq,df) """ BIG = 20.0 diff --git a/experiments/templatetags/experiments.py b/experiments/templatetags/experiments.py index f3dd5f29..b8efcf55 100644 --- a/experiments/templatetags/experiments.py +++ b/experiments/templatetags/experiments.py @@ -14,17 +14,24 @@ @register.inclusion_tag('experiments/goal.html') def experiment_goal(goal_name): - return {'url': reverse('experiment_goal', kwargs={'goal_name': goal_name, 'cache_buster': uuid4()})} + return { + 'url': reverse( + 'experiment_goal', kwargs={ + 'goal_name': goal_name, 'cache_buster': uuid4()})} @register.inclusion_tag('experiments/confirm_human.html', takes_context=True) def experiments_confirm_human(context): request = context.get('request') - return {'confirmed_human': request.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)} + return { + 'confirmed_human': request.session.get( + conf.CONFIRM_HUMAN_SESSION_KEY, False)} class ExperimentNode(template.Node): - def __init__(self, node_list, experiment_name, alternative, weight, user_variable): + def __init__( + self, node_list, experiment_name, alternative, weight, + user_variable): self.node_list = node_list self.experiment_name = experiment_name self.alternative = alternative @@ -80,13 +87,13 @@ def _parse_token_contents(token_contents): def experiment(parser, token): """ Split Testing experiment tag has the following syntax : - + {% experiment %} experiment content goes here {% endexperiment %} - - If the alternative name is neither 'test' nor 'control' an exception is raised - during rendering. + + If the alternative name is neither 'test' nor 'control' an exception is + raised during rendering. """ try: token_contents = token.split_contents() @@ -95,10 +102,13 @@ def experiment(parser, token): node_list = parser.parse(('endexperiment', )) parser.delete_first_token() except ValueError: - raise template.TemplateSyntaxError("Syntax should be like :" - "{% experiment experiment_name alternative [weight=val] [user=val] %}") + raise template.TemplateSyntaxError( + "Syntax should be like :" + "{% experiment experiment_name alternative [weight=val] [user=val]" + " %}") - return ExperimentNode(node_list, experiment_name, alternative, weight, user_variable) + return ExperimentNode( + node_list, experiment_name, alternative, weight, user_variable) @register.assignment_tag(takes_context=True) @@ -107,4 +117,5 @@ def experiment_enroll(context, experiment_name, *alternatives, **kwargs): user = participant(user=kwargs['user']) else: user = participant(request=context.get('request', None)) + return user.enroll(experiment_name, list(alternatives)) diff --git a/experiments/tests/test_admin.py b/experiments/tests/test_admin.py index d8ae6e9a..0b5dfc79 100644 --- a/experiments/tests/test_admin.py +++ b/experiments/tests/test_admin.py @@ -11,51 +11,66 @@ class AdminTestCase(TestCase): def test_set_state(self): - experiment = Experiment.objects.create(name='test_experiment', state=CONTROL_STATE) - User.objects.create_superuser(username='user', email='deleted@mixcloud.com', password='pass') + experiment = Experiment.objects.create( + name='test_experiment', state=CONTROL_STATE) + User.objects.create_superuser( + username='user', email='deleted@mixcloud.com', password='pass') self.client.login(username='user', password='pass') - self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, CONTROL_STATE) - response = self.client.post(reverse('admin:experiment_admin_set_state'), { - 'experiment': experiment.name, - 'state': ENABLED_STATE, - }) + self.assertEqual(Experiment.objects.get( + pk=experiment.pk).state, CONTROL_STATE) + response = self.client.post( + reverse('admin:experiment_admin_set_state'), { + 'experiment': experiment.name, + 'state': ENABLED_STATE}) + self.assertEqual(response.status_code, 200) - self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, ENABLED_STATE) + self.assertEqual( + Experiment.objects.get(pk=experiment.pk).state, ENABLED_STATE) self.assertIsNone(Experiment.objects.get(pk=experiment.pk).end_date) - response = self.client.post(reverse('admin:experiment_admin_set_state'), { - 'experiment': experiment.name, - 'state': CONTROL_STATE, - }) + response = self.client.post( + reverse('admin:experiment_admin_set_state'), { + 'experiment': experiment.name, + 'state': CONTROL_STATE}) + self.assertEqual(response.status_code, 200) - self.assertEqual(Experiment.objects.get(pk=experiment.pk).state, CONTROL_STATE) + self.assertEqual(Experiment.objects.get( + pk=experiment.pk).state, CONTROL_STATE) self.assertIsNotNone(Experiment.objects.get(pk=experiment.pk).end_date) def test_set_alternative(self): - experiment = Experiment.objects.create(name='test_experiment', state=ENABLED_STATE) - user = User.objects.create_superuser(username='user', email='deleted@mixcloud.com', password='pass') + experiment = Experiment.objects.create( + name='test_experiment', state=ENABLED_STATE) + user = User.objects.create_superuser( + username='user', email='deleted@mixcloud.com', password='pass') self.client.login(username='user', password='pass') - participant(user=user).enroll('test_experiment', alternatives=['other1', 'other2']) + participant(user=user).enroll( + 'test_experiment', alternatives=['other1', 'other2']) for alternative in ('other2', 'control', 'other1'): - response = self.client.post(reverse('admin:experiment_admin_set_alternative'), { - 'experiment': experiment.name, - 'alternative': alternative, - }) - self.assertDictEqual(json.loads(response.content.decode('utf-8')), { - 'success': True, - 'alternative': alternative, - }) - self.assertEqual(participant(user=user).get_alternative('test_experiment'), alternative) + response = self.client.post( + reverse('admin:experiment_admin_set_alternative'), { + 'experiment': experiment.name, + 'alternative': alternative}) + self.assertDictEqual( + json.loads(response.content.decode('utf-8')), { + 'success': True, + 'alternative': alternative}) + + self.assertEqual(participant(user=user).get_alternative( + 'test_experiment'), alternative) def test_permissions(self): # redirect to login if not logged in - self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) - self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) + self.assertEqual(302, self.client.post(reverse( + 'admin:experiment_admin_set_state'), {}).status_code) + self.assertEqual(302, self.client.post(reverse( + 'admin:experiment_admin_set_alternative'), {}).status_code) - response = self.client.post(reverse('admin:experiment_admin_set_alternative'), {}) + response = self.client.post( + reverse('admin:experiment_admin_set_alternative'), {}) self.assertEqual(response.status_code, 302) # non staff user @@ -63,17 +78,23 @@ def test_permissions(self): user.save() self.client.login(username='user', password='pass') - self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) - self.assertEqual(302, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) + self.assertEqual(302, self.client.post(reverse( + 'admin:experiment_admin_set_state'), {}).status_code) + self.assertEqual(302, self.client.post(reverse( + 'admin:experiment_admin_set_alternative'), {}).status_code) user.is_staff = True user.save() - self.assertEqual(403, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) - self.assertEqual(403, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) + self.assertEqual(403, self.client.post(reverse( + 'admin:experiment_admin_set_state'), {}).status_code) + self.assertEqual(403, self.client.post(reverse( + 'admin:experiment_admin_set_alternative'), {}).status_code) permission = Permission.objects.get(codename='change_experiment') user.user_permissions.add(permission) - self.assertEqual(400, self.client.post(reverse('admin:experiment_admin_set_state'), {}).status_code) - self.assertEqual(400, self.client.post(reverse('admin:experiment_admin_set_alternative'), {}).status_code) + self.assertEqual(400, self.client.post(reverse( + 'admin:experiment_admin_set_state'), {}).status_code) + self.assertEqual(400, self.client.post(reverse( + 'admin:experiment_admin_set_alternative'), {}).status_code) diff --git a/experiments/tests/test_signals.py b/experiments/tests/test_signals.py index 9beac792..de5cbf10 100644 --- a/experiments/tests/test_signals.py +++ b/experiments/tests/test_signals.py @@ -26,17 +26,20 @@ def signal_handler(self, *args, **kwargs): class SignalsTestCase(TestCase): def setUp(self): - self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) + self.experiment = Experiment.objects.create( + name=EXPERIMENT_NAME, state=ENABLED_STATE) User = get_user_model() self.user = User.objects.create(username='brian') def test_sends_enroll_signal(self): with WatchSignal(user_enrolled) as signal: - participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) + participant(user=self.user).enroll( + EXPERIMENT_NAME, ['red', 'blue']) self.assertTrue(signal.called) def test_does_not_send_enroll_signal_again(self): participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) with WatchSignal(user_enrolled) as signal: - participant(user=self.user).enroll(EXPERIMENT_NAME, ['red', 'blue']) + participant(user=self.user).enroll( + EXPERIMENT_NAME, ['red', 'blue']) self.assertFalse(signal.called) diff --git a/experiments/tests/test_significance.py b/experiments/tests/test_significance.py index 4a41b89c..86d93dd4 100644 --- a/experiments/tests/test_significance.py +++ b/experiments/tests/test_significance.py @@ -13,15 +13,18 @@ def test_empty_sets(self): def test_identical_ranges(self): distribution = dict((x, 1) for x in range(50)) - self.assertUandPCorrect(distribution, distribution, 1250.0, 0.49862467827855483) + self.assertUandPCorrect( + distribution, distribution, 1250.0, 0.49862467827855483) def test_many_repeated_values(self): - self.assertUandPCorrect({0: 100, 1: 50}, {0: 110, 1: 60}, 12500.0, 0.35672951675909859) + self.assertUandPCorrect( + {0: 100, 1: 50}, {0: 110, 1: 60}, 12500.0, 0.35672951675909859) def test_large_range(self): distribution_a = dict((x, 1) for x in range(10000)) distribution_b = dict((x + 1, 1) for x in range(10000)) - self.assertUandPCorrect(distribution_a, distribution_b, 49990000.5, 0.49023014794874586) + self.assertUandPCorrect( + distribution_a, distribution_b, 49990000.5, 0.49023014794874586) def test_very_different_sizes(self): distribution_a = dict((x, 1) for x in range(10000)) @@ -33,27 +36,40 @@ def assertUandPCorrect(self, distribution_a, distribution_b, u, p): self.assertEqual(our_u, u, "U score incorrect") self.assertAlmostEqual(our_p, p, msg="p value incorrect") + class ChiSquare(TestCase): def test_equal(self): self.assertChiSquareCorrect(((100, 10), (200, 20)), 0, 1) - self.assertChiSquareCorrect(((100, 100, 100), (200, 200, 200), (300, 300, 300)), 0, 1) + self.assertChiSquareCorrect((( + 100, 100, 100), (200, 200, 200), (300, 300, 300)), 0, 1) def test_error(self): self.assertRaises(TypeError, chi_square_p_value((1,))) - self.assertRaises(TypeError, chi_square_p_value(((1,2,3)))) + self.assertRaises(TypeError, chi_square_p_value(((1, 2, 3)))) def test_is_none(self): - self.assertEqual(chi_square_p_value(((1, 1), (1, -1))), (None, None), "Negative numbers should not be allowed") - self.assertEqual(chi_square_p_value(((0, 0), (0, 0))), (None, None), "Zero sample size should not be allowed") - self.assertIsNone(chi_square_p_value(((1,), (1, 2))), "Unequal matrices should not be allowed") - self.assertIsNone(chi_square_p_value(((1, 2, 3), (1, 2, 3), (1, 2))), "Unequal matrices should not be allowed") - self.assertIsNone(chi_square_p_value(((100, 10), (200, 20), (300, 30), (400, 40))), "Matrices have to be square") + self.assertEqual(chi_square_p_value((( + 1, 1), (1, -1))), (None, None), + "Negative numbers should not be allowed") + self.assertEqual(chi_square_p_value((( + 0, 0), (0, 0))), (None, None), + "Zero sample size should not be allowed") + self.assertIsNone(chi_square_p_value((( + 1,), (1, 2))), "Unequal matrices should not be allowed") + self.assertIsNone(chi_square_p_value((( + 1, 2, 3), (1, 2, 3), (1, 2))), + "Unequal matrices should not be allowed") + self.assertIsNone(chi_square_p_value((( + 100, 10), (200, 20), (300, 30), (400, 40))), + "Matrices have to be square") def test_stress(self): # Generate a large matrix matrix = [] + for col in range(0, 100): matrix.append([]) + for row in range(0, 100): matrix[col].append(random.randint(0, 10)) @@ -61,15 +77,23 @@ def test_stress(self): def test_accept_hypothesis(self): self.assertChiSquareCorrect(((36, 14), (30, 25)), 3.418, 0.065, 3) - self.assertChiSquareCorrect(((100, 50), (210, 110)), 0.04935, 0.8242, 3) - self.assertChiSquareCorrect(((100, 50, 10), (110, 50, 10), (140, 55, 11)), 1.2238, 0.8741, 3) + self.assertChiSquareCorrect((( + 100, 50), (210, 110)), 0.04935, 0.8242, 3) + self.assertChiSquareCorrect((( + 100, 50, 10), (110, 50, 10), (140, 55, 11)), 1.2238, 0.8741, 3) def test_reject_hypothesis(self): - self.assertChiSquareCorrect(((100, 20), (200, 20)), 4.2929, 0.0383, 4) - self.assertChiSquareCorrect(((100, 50, 10), (110, 70, 20), (140, 55, 6)), 13.0217, 0.0111, 3) - - def assertChiSquareCorrect(self, matrix, observed_test_statistic, p_value, accuracy=7): - observed_test_statistic_result, p_value_result = chi_square_p_value(matrix) - self.assertAlmostEqual(observed_test_statistic_result, observed_test_statistic, accuracy, 'Wrong observed result') - self.assertAlmostEqual(p_value_result, p_value, accuracy, 'Wrong P Value') - + self.assertChiSquareCorrect((( + 100, 20), (200, 20)), 4.2929, 0.0383, 4) + self.assertChiSquareCorrect((( + 100, 50, 10), (110, 70, 20), (140, 55, 6)), 13.0217, 0.0111, 3) + + def assertChiSquareCorrect( + self, matrix, observed_test_statistic, p_value, accuracy=7): + observed_test_statistic_result, p_value_result = chi_square_p_value( + matrix) + self.assertAlmostEqual( + observed_test_statistic_result, observed_test_statistic, accuracy, + 'Wrong observed result') + self.assertAlmostEqual( + p_value_result, p_value, accuracy, 'Wrong P Value') diff --git a/experiments/tests/test_templatetags.py b/experiments/tests/test_templatetags.py index 9e3aee2d..54d72c1e 100644 --- a/experiments/tests/test_templatetags.py +++ b/experiments/tests/test_templatetags.py @@ -27,19 +27,23 @@ def test_handles_labelled_weight(self): self.assertEqual(weight, '10') def test_handles_user(self): - token_contents = ('experiment', 'backgroundcolor', 'blue', 'user=commenter') + token_contents = ( + 'experiment', 'backgroundcolor', 'blue', 'user=commenter') experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) self.assertEqual(user_resolvable.var, 'commenter') def test_handles_user_and_weight(self): - token_contents = ('experiment', 'backgroundcolor', 'blue', 'user=commenter', 'weight=10') + token_contents = ( + 'experiment', 'backgroundcolor', 'blue', 'user=commenter', + 'weight=10') experiment_name, alternative, weight, user_resolvable = _parse_token_contents(token_contents) self.assertEqual(user_resolvable.var, 'commenter') self.assertEqual(weight, '10') def test_raises_on_insufficient_arguments(self): token_contents = ('experiment', 'backgroundcolor') - self.assertRaises(ValueError, lambda: _parse_token_contents(token_contents)) + self.assertRaises(ValueError, lambda: _parse_token_contents( + token_contents)) class ExperimentAutoCreateTestCase(TestCase): @@ -48,21 +52,27 @@ def test_template_auto_create_off(self): request = RequestFactory().get('/') request.user = User.objects.create(username='test') Template("{% load experiments %}{% experiment test_experiment control %}{% endexperiment %}").render(Context({'request': request})) - self.assertFalse(Experiment.objects.filter(name="test_experiment").exists()) + self.assertFalse(Experiment.objects.filter( + name="test_experiment").exists()) def test_template_auto_create_on(self): request = RequestFactory().get('/') request.user = User.objects.create(username='test') Template("{% load experiments %}{% experiment test_experiment control %}{% endexperiment %}").render(Context({'request': request})) - self.assertTrue(Experiment.objects.filter(name="test_experiment").exists()) + self.assertTrue(Experiment.objects.filter( + name="test_experiment").exists()) @override_settings(EXPERIMENTS_AUTO_CREATE=False) def test_view_auto_create_off(self): user = User.objects.create(username='test') - participant(user=user).enroll('test_experiment_y', alternatives=['other']) - self.assertFalse(Experiment.objects.filter(name="test_experiment_y").exists()) + participant(user=user).enroll( + 'test_experiment_y', alternatives=['other']) + self.assertFalse(Experiment.objects.filter( + name="test_experiment_y").exists()) def test_view_auto_create_on(self): user = User.objects.create(username='test') - participant(user=user).enroll('test_experiment_x', alternatives=['other']) - self.assertTrue(Experiment.objects.filter(name="test_experiment_x").exists()) + participant(user=user).enroll( + 'test_experiment_x', alternatives=['other']) + self.assertTrue(Experiment.objects.filter( + name="test_experiment_x").exists()) diff --git a/experiments/tests/test_webuser.py b/experiments/tests/test_webuser.py index 5a09c254..6c86eca8 100644 --- a/experiments/tests/test_webuser.py +++ b/experiments/tests/test_webuser.py @@ -1,16 +1,18 @@ from __future__ import absolute_import - from datetime import timedelta -from django.http import HttpResponse +import random +from django.http import HttpResponse from django.test import TestCase from django.test.client import RequestFactory from django.contrib.auth.models import AnonymousUser from django.contrib.auth import get_user_model from django.contrib.sessions.backends.db import SessionStore as DatabaseSession from django.utils import timezone -from experiments import conf +from mock import patch + +from experiments import conf from experiments.experiment_counters import ExperimentCounter from experiments.middleware import ExperimentsRetentionMiddleware from experiments.models import Experiment, ENABLED_STATE, Enrollment @@ -18,10 +20,6 @@ from experiments.signal_handlers import transfer_enrollments_to_user from experiments.utils import participant -from mock import patch - -import random - request_factory = RequestFactory() TEST_ALTERNATIVE = 'blue' @@ -42,21 +40,26 @@ def tearDown(self): def test_enrollment_initially_control(self): experiment_user = participant(self.request) - self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") + self.assertEqual(experiment_user.get_alternative( + EXPERIMENT_NAME), 'control', "Default Enrollment wasn't control") def test_user_enrolls(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) - self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") + self.assertEqual(experiment_user.get_alternative( + EXPERIMENT_NAME), TEST_ALTERNATIVE, "Wrong Alternative Set") def test_record_goal_increments_counts(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) + self.assertEqual(self.experiment_counter.goal_count( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0) experiment_user.goal(TEST_GOAL) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment Goal count") + self.assertEqual(self.experiment_counter.goal_count( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, + "Did not increment Goal count") def test_can_record_goal_multiple_times(self): experiment_user = participant(self.request) @@ -66,15 +69,21 @@ def test_can_record_goal_multiple_times(self): experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) experiment_user.goal(TEST_GOAL) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not increment goal count correctly") - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, "Incorrect goal count distribution") + self.assertEqual(self.experiment_counter.goal_count( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, + "Did not increment goal count correctly") + self.assertEqual(self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), {3: 1}, + "Incorrect goal count distribution") def test_counts_increment_immediately_once_confirmed_human(self): experiment_user = participant(self.request) experiment_user.confirm_human() experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") + self.assertEqual(self.experiment_counter.participant_count( + self.experiment, TEST_ALTERNATIVE), 1, + "Did not count participant after confirm human") def test_visit_increases_goal(self): thetime = timezone.now() @@ -84,13 +93,32 @@ def test_visit_increases_goal(self): experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.visit() - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Not Present Visit was not correctly counted") - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") - with patch('experiments.utils.now', return_value=thetime + timedelta(hours=7)): + self.assertEqual(self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, + VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, + "Not Present Visit was not correctly counted") + self.assertEqual( + self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, + VISIT_PRESENT_COUNT_GOAL), {}, + "Present Visit was not correctly counted") + + with patch( + 'experiments.utils.now', + return_value=thetime + timedelta(hours=7)): experiment_user.visit() - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, "No Present Visit was not correctly counted") - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {1: 1}, "Present Visit was not correctly counted") + + self.assertEqual( + self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, + VISIT_NOT_PRESENT_COUNT_GOAL), {2: 1}, + "No Present Visit was not correctly counted") + self.assertEqual( + self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, + VISIT_PRESENT_COUNT_GOAL), {1: 1}, + "Present Visit was not correctly counted") def test_visit_twice_increases_once(self): experiment_user = participant(self.request) @@ -100,13 +128,22 @@ def test_visit_twice_increases_once(self): experiment_user.visit() experiment_user.visit() - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, "Visit was not correctly counted") - self.assertEqual(self.experiment_counter.goal_distribution(self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, "Present Visit was not correctly counted") + self.assertEqual( + self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, + VISIT_NOT_PRESENT_COUNT_GOAL), {1: 1}, + "Visit was not correctly counted") + self.assertEqual(self.experiment_counter.goal_distribution( + self.experiment, TEST_ALTERNATIVE, VISIT_PRESENT_COUNT_GOAL), {}, + "Present Visit was not correctly counted") def test_user_force_enrolls(self): experiment_user = participant(self.request) - experiment_user.enroll(EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], force_alternative='alternative2') - self.assertEqual(experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') + experiment_user.enroll( + EXPERIMENT_NAME, ['control', 'alternative1', 'alternative2'], + force_alternative='alternative2') + self.assertEqual( + experiment_user.get_alternative(EXPERIMENT_NAME), 'alternative2') def test_user_does_not_force_enroll_to_new_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] @@ -115,20 +152,27 @@ def test_user_does_not_force_enroll_to_new_alternative(self): alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) - other_alternative = random.choice(list(set(alternatives) - set(alternative))) - experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) - self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) + other_alternative = random.choice( + list(set(alternatives) - set(alternative))) + experiment_user.enroll( + EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) + self.assertEqual(alternative, experiment_user.get_alternative( + EXPERIMENT_NAME)) def test_second_force_enroll_does_not_change_alternative(self): alternatives = ['control', 'alternative1', 'alternative2'] experiment_user = participant(self.request) - experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative='alternative1') + experiment_user.enroll( + EXPERIMENT_NAME, alternatives, force_alternative='alternative1') alternative = experiment_user.get_alternative(EXPERIMENT_NAME) self.assertIsNotNone(alternative) - other_alternative = random.choice(list(set(alternatives) - set(alternative))) - experiment_user.enroll(EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) - self.assertEqual(alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) + other_alternative = random.choice( + list(set(alternatives) - set(alternative))) + experiment_user.enroll( + EXPERIMENT_NAME, alternatives, force_alternative=other_alternative) + self.assertEqual( + alternative, experiment_user.get_alternative(EXPERIMENT_NAME)) class WebUserAnonymousTestCase(WebUserTests, TestCase): @@ -141,18 +185,30 @@ def test_confirm_human_increments_participant_count(self): experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Counted participant before confirmed human") + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, TEST_ALTERNATIVE), 0, + "Counted participant before confirmed human") experiment_user.confirm_human() - self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 1, "Did not count participant after confirm human") + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, TEST_ALTERNATIVE), 1, + "Did not count participant after confirm human") def test_confirm_human_increments_goal_count(self): experiment_user = participant(self.request) experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) experiment_user.goal(TEST_GOAL) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0, "Counted goal before confirmed human") + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 0, + "Counted goal before confirmed human") experiment_user.confirm_human() - self.assertEqual(self.experiment_counter.goal_count(self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, "Did not count goal after confirm human") + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, TEST_ALTERNATIVE, TEST_GOAL), 1, + "Did not count goal after confirm human") class WebUserAuthenticatedTestCase(WebUserTests, TestCase): @@ -165,24 +221,39 @@ def setUp(self): class BotTests(object): def setUp(self): - self.experiment = Experiment(name='backgroundcolor', state=ENABLED_STATE) + self.experiment = Experiment( + name='backgroundcolor', state=ENABLED_STATE) self.experiment.save() self.experiment_counter = ExperimentCounter() def test_user_does_not_enroll(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, TEST_ALTERNATIVE), 0, + "Bot counted towards results") def test_user_does_not_fire_goals(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) self.experiment_user.goal(TEST_GOAL) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, TEST_ALTERNATIVE), 0, "Bot counted towards results") + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, TEST_ALTERNATIVE), 0, + "Bot counted towards results") def test_bot_in_control_group(self): self.experiment_user.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) - self.assertEqual(self.experiment_user.get_alternative(EXPERIMENT_NAME), 'control', "Bot enrolled in a group") - self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, TEST_ALTERNATIVE), False, "Bot in test alternative") - self.assertEqual(self.experiment_user.is_enrolled(self.experiment.name, CONTROL_GROUP), True, "Bot not in control group") + self.assertEqual( + self.experiment_user.get_alternative( + EXPERIMENT_NAME), 'control', "Bot enrolled in a group") + self.assertEqual( + self.experiment_user.is_enrolled( + self.experiment.name, TEST_ALTERNATIVE), False, + "Bot in test alternative") + self.assertEqual( + self.experiment_user.is_enrolled( + self.experiment.name, CONTROL_GROUP), True, + "Bot not in control group") def tearDown(self): self.experiment_counter.delete(self.experiment) @@ -191,7 +262,8 @@ def tearDown(self): class LoggedOutBotTestCase(BotTests, TestCase): def setUp(self): super(LoggedOutBotTestCase, self).setUp() - self.request = request_factory.get('/', HTTP_USER_AGENT='GoogleBot/2.1') + self.request = request_factory.get( + '/', HTTP_USER_AGENT='GoogleBot/2.1') self.experiment_user = participant(self.request) @@ -208,7 +280,8 @@ def setUp(self): class ParticipantCacheTestCase(TestCase): def setUp(self): - self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) + self.experiment = Experiment.objects.create( + name='test_experiment1', state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() def tearDown(self): @@ -226,16 +299,19 @@ def test_transfer_enrollments(self): # if the participant cache hasn't been wiped appropriately then the # session experiment user will be impacted instead of the authenticated # experiment user - ExperimentsRetentionMiddleware().process_response(request, HttpResponse()) + ExperimentsRetentionMiddleware().process_response( + request, HttpResponse()) self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) class ConfirmHumanTestCase(TestCase): def setUp(self): - self.experiment = Experiment.objects.create(name='test_experiment1', state=ENABLED_STATE) + self.experiment = Experiment.objects.create( + name='test_experiment1', state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() self.experiment_user = participant(session=DatabaseSession()) - self.alternative = self.experiment_user.enroll(self.experiment.name, ['alternative']) + self.alternative = self.experiment_user.enroll( + self.experiment.name, ['alternative']) self.experiment_user.goal('my_goal') def tearDown(self): @@ -243,28 +319,48 @@ def tearDown(self): def test_confirm_human_updates_experiment(self): self.assertIn('experiments_goals', self.experiment_user.session) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, self.alternative), 0) + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.assertNotIn('experiments_goals', self.experiment_user.session) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, self.alternative), 1) + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_called_twice(self): """ Ensuring that counters aren't incremented twice """ - self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 0) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 0) + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, self.alternative), 0) + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, self.alternative, 'my_goal'), 0) self.experiment_user.confirm_human() self.experiment_user.confirm_human() - self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) + self.assertEqual( + self.experiment_counter.participant_count( + self.experiment, self.alternative), 1) + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, self.alternative, 'my_goal'), 1) def test_confirm_human_sets_session(self): - self.assertFalse(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) + self.assertFalse( + self.experiment_user.session.get( + conf.CONFIRM_HUMAN_SESSION_KEY, False)) self.experiment_user.confirm_human() - self.assertTrue(self.experiment_user.session.get(conf.CONFIRM_HUMAN_SESSION_KEY, False)) + self.assertTrue( + self.experiment_user.session.get( + conf.CONFIRM_HUMAN_SESSION_KEY, False)) def test_session_already_confirmed(self): """ @@ -272,8 +368,11 @@ def test_session_already_confirmed(self): """ self.experiment_user.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True self.experiment_user.confirm_human() - self.assertEqual(self.experiment_counter.participant_count(self.experiment, self.alternative), 1) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, self.alternative, 'my_goal'), 1) + self.assertEqual(self.experiment_counter.participant_count( + self.experiment, self.alternative), 1) + self.assertEqual( + self.experiment_counter.goal_count( + self.experiment, self.alternative, 'my_goal'), 1) class DefaultAlternativeTestCase(TestCase): @@ -283,10 +382,16 @@ def test_default_alternative(self): experiment.ensure_alternative_exists('alt1') experiment.ensure_alternative_exists('alt2') - self.assertEqual(conf.CONTROL_GROUP, participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) + self.assertEqual( + conf.CONTROL_GROUP, participant(session=DatabaseSession()).enroll( + 'test_default', ['alt1', 'alt2'])) experiment.set_default_alternative('alt2') experiment.save() - self.assertEqual('alt2', participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) + self.assertEqual( + 'alt2', participant(session=DatabaseSession()).enroll( + 'test_default', ['alt1', 'alt2'])) experiment.set_default_alternative('alt1') experiment.save() - self.assertEqual('alt1', participant(session=DatabaseSession()).enroll('test_default', ['alt1', 'alt2'])) + self.assertEqual( + 'alt1', participant(session=DatabaseSession()).enroll( + 'test_default', ['alt1', 'alt2'])) diff --git a/experiments/tests/test_webuser_incorporate.py b/experiments/tests/test_webuser_incorporate.py index 28470e98..0edc4881 100644 --- a/experiments/tests/test_webuser_incorporate.py +++ b/experiments/tests/test_webuser_incorporate.py @@ -1,18 +1,19 @@ +from unittest import TestSuite + from django.http import HttpResponse from django.test import TestCase, RequestFactory from django.contrib.sessions.backends.db import SessionStore as DatabaseSession +from django.contrib.auth import get_user_model -from unittest import TestSuite from experiments import conf from experiments.experiment_counters import ExperimentCounter from experiments.middleware import ExperimentsRetentionMiddleware from experiments.signal_handlers import transfer_enrollments_to_user -from experiments.utils import DummyUser, SessionUser, AuthenticatedUser, participant +from experiments.utils import ( + DummyUser, SessionUser, AuthenticatedUser, participant) from experiments.models import Experiment, ENABLED_STATE, Enrollment -from django.contrib.auth import get_user_model - TEST_ALTERNATIVE = 'blue' EXPERIMENT_NAME = 'backgroundcolor' @@ -30,15 +31,19 @@ def test_incorporates_enrollment_from_other(self): return try: - experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) - self.incorporated.set_alternative(EXPERIMENT_NAME, TEST_ALTERNATIVE) + experiment = Experiment.objects.create( + name=EXPERIMENT_NAME, state=ENABLED_STATE) + self.incorporated.set_alternative( + EXPERIMENT_NAME, TEST_ALTERNATIVE) self.incorporating.incorporate(self.incorporated) - self.assertEqual(self.incorporating.get_alternative(EXPERIMENT_NAME), TEST_ALTERNATIVE) + self.assertEqual(self.incorporating.get_alternative( + EXPERIMENT_NAME), TEST_ALTERNATIVE) finally: self.experiment_counter.delete(experiment) def _has_data(self): - return not isinstance(self.incorporated, DummyUser) and not isinstance(self.incorporating, DummyUser) + return not isinstance(self.incorporated, DummyUser) and not isinstance( + self.incorporating, DummyUser) def dummy(incorporating): @@ -51,7 +56,9 @@ def anonymous(incorporating): def authenticated(incorporating): User = get_user_model() - return AuthenticatedUser(user=User.objects.create(username=['incorporating_user', 'incorporated_user'][incorporating])) + return AuthenticatedUser(user=User.objects.create( + username=['incorporating_user', 'incorporated_user'][incorporating])) + user_factories = (dummy, anonymous, authenticated) @@ -76,12 +83,14 @@ def setUp(self): self.incorporating = incorporating(True) self.incorporated = incorporated(False) InstantiatedTestCase.__name__ = "WebUserIncorporateTestCase_into_%s_from_%s" % (incorporating.__name__, incorporated.__name__) + return InstantiatedTestCase class IncorporateTestCase(TestCase): def setUp(self): - self.experiment = Experiment.objects.create(name=EXPERIMENT_NAME, state=ENABLED_STATE) + self.experiment = Experiment.objects.create( + name=EXPERIMENT_NAME, state=ENABLED_STATE) self.experiment_counter = ExperimentCounter() User = get_user_model() @@ -101,14 +110,17 @@ def _login(self): transfer_enrollments_to_user(None, self.request, self.user) def test_visit_incorporate(self): - alternative = participant(self.request).enroll(self.experiment.name, ['alternative']) + alternative = participant(self.request).enroll( + self.experiment.name, ['alternative']) - ExperimentsRetentionMiddleware().process_response(self.request, HttpResponse()) + ExperimentsRetentionMiddleware().process_response( + self.request, HttpResponse()) self.assertEqual( - dict(self.experiment_counter.participant_goal_frequencies(self.experiment, - alternative, - participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], + dict(self.experiment_counter.participant_goal_frequencies( + self.experiment, alternative, + participant(self.request)._participant_identifier()))[ + conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1 ) @@ -118,11 +130,14 @@ def test_visit_incorporate(self): self.assertTrue(Enrollment.objects.all().exists()) self.assertIsNotNone(Enrollment.objects.all()[0].last_seen) self.assertEqual( - dict(self.experiment_counter.participant_goal_frequencies(self.experiment, - alternative, - participant(self.request)._participant_identifier()))[conf.VISIT_NOT_PRESENT_COUNT_GOAL], + dict(self.experiment_counter.participant_goal_frequencies( + self.experiment, alternative, + participant(self.request)._participant_identifier()))[ + conf.VISIT_NOT_PRESENT_COUNT_GOAL], 1 ) - self.assertEqual(self.experiment_counter.goal_count(self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), 1) - self.assertEqual(self.experiment_counter.participant_count(self.experiment, alternative), 1) - + self.assertEqual(self.experiment_counter.goal_count( + self.experiment, alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL), + 1) + self.assertEqual(self.experiment_counter.participant_count( + self.experiment, alternative), 1) diff --git a/experiments/urls.py b/experiments/urls.py index 580f4a49..b80af44f 100644 --- a/experiments/urls.py +++ b/experiments/urls.py @@ -2,7 +2,13 @@ from experiments import views urlpatterns = [ - url(r'^goal/(?P[^/]+)/(?P[^/]+)?$', views.record_experiment_goal, name="experiment_goal"), - url(r'^confirm_human/$', views.confirm_human, name="experiment_confirm_human"), - url(r'^change_alternative/(?P[a-zA-Z0-9-_]+)/(?P[a-zA-Z0-9-_]+)/$', views.change_alternative, name="experiment_change_alternative"), + url( + r'^goal/(?P[^/]+)/(?P[^/]+)?$', + views.record_experiment_goal, name="experiment_goal"), + url( + r'^confirm_human/$', views.confirm_human, + name="experiment_confirm_human"), + url( + r'^change_alternative/(?P[a-zA-Z0-9-_]+)/(?P[a-zA-Z0-9-_]+)/$', + views.change_alternative, name="experiment_change_alternative"), ] diff --git a/experiments/utils.py b/experiments/utils.py index 05529b35..c56b29a4 100644 --- a/experiments/utils.py +++ b/experiments/utils.py @@ -1,3 +1,10 @@ +import collections +import numbers +import logging +import json +from collections import namedtuple +from datetime import timedelta + from django.db import IntegrityError from experiments.models import Enrollment @@ -7,26 +14,23 @@ from experiments.experiment_counters import ExperimentCounter from experiments import conf -from collections import namedtuple -from datetime import timedelta - -import collections -import numbers -import logging -import json logger = logging.getLogger('experiments') def participant(request=None, session=None, user=None): - # This caches the experiment user on the request object because AuthenticatedUser can involve database lookups that - # it caches. Signals are attached to login/logout to clear the cache using clear_participant_cache + # This caches the experiment user on the request object because + # AuthenticatedUser can involve database lookups that it caches. Signals + # are attached to login/logout to clear the cache using + # clear_participant_cache if request and hasattr(request, '_experiments_user'): return request._experiments_user else: result = _get_participant(request, session, user) + if request: request._experiments_user = result + return result @@ -41,7 +45,8 @@ def _get_participant(request, session, user): if request and hasattr(request, 'session') and not session: session = request.session - if request and conf.BOT_REGEX.search(request.META.get("HTTP_USER_AGENT", "")): + if request and conf.BOT_REGEX.search( + request.META.get("HTTP_USER_AGENT", "")): return DummyUser() elif user and user.is_authenticated(): if getattr(user, 'is_confirmed_human', True): @@ -54,7 +59,9 @@ def _get_participant(request, session, user): return DummyUser() -EnrollmentData = namedtuple('EnrollmentData', ['experiment', 'alternative', 'enrollment_date', 'last_seen']) +EnrollmentData = namedtuple( + 'EnrollmentData', [ + 'experiment', 'alternative', 'enrollment_date', 'last_seen']) class WebUser(object): @@ -77,11 +84,16 @@ def enroll(self, experiment_name, alternatives, force_alternative=None): if experiment.is_displaying_alternatives(): if isinstance(alternatives, collections.Mapping): if conf.CONTROL_GROUP not in alternatives: - experiment.ensure_alternative_exists(conf.CONTROL_GROUP, 1) + experiment.ensure_alternative_exists( + conf.CONTROL_GROUP, 1) + for alternative, weight in alternatives.items(): - experiment.ensure_alternative_exists(alternative, weight) + experiment.ensure_alternative_exists( + alternative, weight) else: - alternatives_including_control = alternatives + [conf.CONTROL_GROUP] + alternatives_including_control = alternatives + [ + conf.CONTROL_GROUP] + for alternative in alternatives_including_control: experiment.ensure_alternative_exists(alternative) @@ -105,13 +117,16 @@ def get_alternative(self, experiment_name): """ experiment = None try: - # catching the KeyError instead of using .get so that the experiment is auto created if desired + # catching the KeyError instead of using .get so that the + # experiment is auto created if desired experiment = experiment_manager[experiment_name] except KeyError: pass + if experiment: if experiment.is_displaying_alternatives(): alternative = self._get_enrollment(experiment) + if alternative is not None: return alternative else: @@ -119,11 +134,14 @@ def get_alternative(self, experiment_name): return conf.CONTROL_GROUP def set_alternative(self, experiment_name, alternative): - """Explicitly set the alternative the user is enrolled in for the specified experiment. + """Explicitly set the alternative the user is enrolled in for the + specified experiment. - This allows you to change a user between alternatives. The user and goal counts for the new - alternative will be increment, but those for the old one will not be decremented. The user will - be enrolled in the experiment even if the experiment would not normally accept this user.""" + This allows you to change a user between alternatives. The user and + goal counts for the new alternative will be increment, but those for + the old one will not be decremented. The user will be enrolled in the + experiment even if the experiment would not normally accept this user. + """ experiment = experiment_manager.get_experiment(experiment_name) if experiment: self._set_enrollment(experiment, alternative) @@ -131,10 +149,14 @@ def set_alternative(self, experiment_name, alternative): def goal(self, goal_name, count=1): """Record that this user has performed a particular goal - This will update the goal stats for all experiments the user is enrolled in.""" + This will update the goal stats for all experiments the user is + enrolled in. + """ for enrollment in self._get_all_enrollments(): if enrollment.experiment.is_displaying_alternatives(): - self._experiment_goal(enrollment.experiment, enrollment.alternative, goal_name, count) + self._experiment_goal( + enrollment.experiment, enrollment.alternative, goal_name, + count) def confirm_human(self): """Mark that this is a real human being (not a bot) and thus results should be counted""" @@ -147,33 +169,52 @@ def incorporate(self, other_user): other user are incorporated. For experiments this user is already enrolled in the results of the other user are discarded. - This takes a relatively large amount of time for each experiment the other - user is enrolled in.""" + This takes a relatively large amount of time for each experiment the + other user is enrolled in. + """ for enrollment in other_user._get_all_enrollments(): if not self._get_enrollment(enrollment.experiment): - self._set_enrollment(enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen) - goals = self.experiment_counter.participant_goal_frequencies(enrollment.experiment, enrollment.alternative, other_user._participant_identifier()) + self._set_enrollment( + enrollment.experiment, enrollment.alternative, + enrollment.enrollment_date, enrollment.last_seen) + goals = self.experiment_counter.participant_goal_frequencies( + enrollment.experiment, enrollment.alternative, + other_user._participant_identifier()) + for goal_name, count in goals: - self.experiment_counter.increment_goal_count(enrollment.experiment, enrollment.alternative, goal_name, self._participant_identifier(), count) + self.experiment_counter.increment_goal_count( + enrollment.experiment, enrollment.alternative, + goal_name, self._participant_identifier(), count) other_user._cancel_enrollment(enrollment.experiment) def visit(self): """Record that the user has visited the site for the purposes of retention tracking""" for enrollment in self._get_all_enrollments(): if enrollment.experiment.is_displaying_alternatives(): - # We have two different goals, VISIT_NOT_PRESENT_COUNT_GOAL and VISIT_PRESENT_COUNT_GOAL. - # VISIT_PRESENT_COUNT_GOAL will avoid firing on the first time we set last_seen as it is assumed that the user is - # on the page and therefore it would automatically trigger and be valueless. - # This should be used for experiments when we enroll the user as part of the pageview, - # alternatively we can use the NOT_PRESENT GOAL which will increment on the first pageview, - # this is mainly useful for notification actions when the users isn't initially present. + # We have two different goals, VISIT_NOT_PRESENT_COUNT_GOAL and + # VISIT_PRESENT_COUNT_GOAL. VISIT_PRESENT_COUNT_GOAL will avoid + # firing on the first time we set last_seen as it is assumed + # that the user is on the page and therefore it would + # automatically trigger and be valueless. This should be used + # for experiments when we enroll the user as part of the + # pageview, alternatively we can use the NOT_PRESENT GOAL which + # will increment on the first pageview, this is mainly useful + # for notification actions when the users isn't initially + # present. if not enrollment.last_seen: - self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) + self._experiment_goal( + enrollment.experiment, enrollment.alternative, + conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) self._set_last_seen(enrollment.experiment, now()) - elif now() - enrollment.last_seen >= timedelta(hours=conf.SESSION_LENGTH): - self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) - self._experiment_goal(enrollment.experiment, enrollment.alternative, conf.VISIT_PRESENT_COUNT_GOAL, 1) + elif now() - enrollment.last_seen >= timedelta( + hours=conf.SESSION_LENGTH): + self._experiment_goal( + enrollment.experiment, enrollment.alternative, + conf.VISIT_NOT_PRESENT_COUNT_GOAL, 1) + self._experiment_goal( + enrollment.experiment, enrollment.alternative, + conf.VISIT_PRESENT_COUNT_GOAL, 1) self._set_last_seen(enrollment.experiment, now()) def _get_enrollment(self, experiment): @@ -182,11 +223,16 @@ def _get_enrollment(self, experiment): `experiment` is an instance of Experiment. If the user is not currently enrolled returns None.""" raise NotImplementedError - def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): - """Explicitly set the alternative the user is enrolled in for the specified experiment. + def _set_enrollment( + self, experiment, alternative, enrollment_date=None, + last_seen=None): + """Explicitly set the alternative the user is enrolled in for the + specified experiment. - This allows you to change a user between alternatives. The user and goal counts for the new - alternative will be increment, but those for the old one will not be decremented.""" + This allows you to change a user between alternatives. The user and + goal counts for the new alternative will be increment, but those for + the old one will not be decremented. + """ raise NotImplementedError def is_enrolled(self, experiment_name, alternative): @@ -224,7 +270,9 @@ class DummyUser(WebUser): def _get_enrollment(self, experiment): return None - def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): + def _set_enrollment( + self, experiment, alternative, enrollment_date=None, + last_seen=None): pass def is_enrolled(self, experiment_name, alternative): @@ -266,22 +314,30 @@ def __init__(self, user, request=None): def _get_enrollment(self, experiment): if experiment.name not in self._enrollment_cache: try: - self._enrollment_cache[experiment.name] = Enrollment.objects.get(user=self.user, experiment=experiment).alternative + self._enrollment_cache[ + experiment.name] = Enrollment.objects.get( + user=self.user, experiment=experiment).alternative except Enrollment.DoesNotExist: self._enrollment_cache[experiment.name] = None + return self._enrollment_cache[experiment.name] - def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): + def _set_enrollment( + self, experiment, alternative, enrollment_date=None, + last_seen=None): if experiment.name in self._enrollment_cache: del self._enrollment_cache[experiment.name] try: - enrollment, _ = Enrollment.objects.get_or_create(user=self.user, experiment=experiment, defaults={'alternative': alternative}) + enrollment, _ = Enrollment.objects.get_or_create( + user=self.user, experiment=experiment, defaults={ + 'alternative': alternative}) except IntegrityError: # Already registered (db race condition under high load) return # Update alternative if it doesn't match enrollment_changed = False + if enrollment.alternative != alternative: enrollment.alternative = alternative enrollment_changed = True @@ -295,48 +351,64 @@ def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_se if enrollment_changed: enrollment.save() - self.experiment_counter.increment_participant_count(experiment, alternative, self._participant_identifier()) - - user_enrolled.send(self, experiment=experiment.name, alternative=alternative, user=self.user, session=None) + self.experiment_counter.increment_participant_count( + experiment, alternative, self._participant_identifier()) + user_enrolled.send( + self, experiment=experiment.name, alternative=alternative, + user=self.user, session=None) def _participant_identifier(self): return 'user:%d' % (self.user.pk, ) def _get_all_enrollments(self): - enrollments = Enrollment.objects.filter(user=self.user).select_related("experiment") + enrollments = Enrollment.objects.filter( + user=self.user).select_related("experiment") + if enrollments: for enrollment in enrollments: - yield EnrollmentData(enrollment.experiment, enrollment.alternative, enrollment.enrollment_date, enrollment.last_seen) + yield EnrollmentData( + enrollment.experiment, enrollment.alternative, + enrollment.enrollment_date, enrollment.last_seen) def _cancel_enrollment(self, experiment): try: - enrollment = Enrollment.objects.get(user=self.user, experiment=experiment) + enrollment = Enrollment.objects.get( + user=self.user, experiment=experiment) except Enrollment.DoesNotExist: pass else: - self.experiment_counter.remove_participant(experiment, enrollment.alternative, self._participant_identifier()) + self.experiment_counter.remove_participant( + experiment, enrollment.alternative, + self._participant_identifier()) enrollment.delete() def _experiment_goal(self, experiment, alternative, goal_name, count): - self.experiment_counter.increment_goal_count(experiment, alternative, goal_name, self._participant_identifier(), count) + self.experiment_counter.increment_goal_count( + experiment, alternative, goal_name, self._participant_identifier(), + count) def _set_last_seen(self, experiment, last_seen): - Enrollment.objects.filter(user=self.user, experiment=experiment).update(last_seen=last_seen) + Enrollment.objects.filter( + user=self.user, experiment=experiment).update(last_seen=last_seen) def _session_enrollment_latest_version(data): try: alternative, unused, enrollment_date, last_seen = data + if isinstance(enrollment_date, numbers.Number): enrollment_date = datetime_from_timestamp(enrollment_date) + if isinstance(last_seen, numbers.Number): last_seen = datetime_from_timestamp(last_seen) + if last_seen: last_seen = fix_awareness(last_seen) except ValueError: # Data from previous version alternative, unused = data enrollment_date = None last_seen = None + return alternative, unused, enrollment_date, last_seen @@ -348,37 +420,60 @@ def __init__(self, session, request=None): def _get_enrollment(self, experiment): enrollments = self.session.get('experiments_enrollments', None) + if enrollments and experiment.name in enrollments: - alternative, _, _, _ = _session_enrollment_latest_version(enrollments[experiment.name]) + alternative, _, _, _ = _session_enrollment_latest_version( + enrollments[experiment.name]) return alternative + return None - def _set_enrollment(self, experiment, alternative, enrollment_date=None, last_seen=None): + def _set_enrollment( + self, experiment, alternative, enrollment_date=None, + last_seen=None): enrollments = self.session.get('experiments_enrollments', {}) - enrollments[experiment.name] = (alternative, None, timestamp_from_datetime(enrollment_date or now()), timestamp_from_datetime(last_seen)) + enrollments[experiment.name] = ( + alternative, None, timestamp_from_datetime( + enrollment_date or now()), timestamp_from_datetime(last_seen)) self.session['experiments_enrollments'] = enrollments + if self._is_verified_human(): - self.experiment_counter.increment_participant_count(experiment, alternative, self._participant_identifier()) + self.experiment_counter.increment_participant_count( + experiment, alternative, self._participant_identifier()) else: - logger.info(json.dumps({'type':'participant_unconfirmed', 'experiment': experiment.name, 'alternative': alternative, 'participant': self._participant_identifier()})) + logger.info( + json.dumps({ + 'type': 'participant_unconfirmed', + 'experiment': experiment.name, + 'alternative': alternative, + 'participant': self._participant_identifier()})) user_enrolled.send(self, experiment=experiment.name, alternative=alternative, user=None, session=self.session) def confirm_human(self): self.session[conf.CONFIRM_HUMAN_SESSION_KEY] = True - logger.info(json.dumps({'type': 'confirm_human', 'participant': self._participant_identifier()})) + logger.info( + json.dumps({ + 'type': 'confirm_human', + 'participant': self._participant_identifier()})) # Replay enrollments for enrollment in self._get_all_enrollments(): - self.experiment_counter.increment_participant_count(enrollment.experiment, enrollment.alternative, self._participant_identifier()) + self.experiment_counter.increment_participant_count( + enrollment.experiment, enrollment.alternative, + self._participant_identifier()) # Replay goals if 'experiments_goals' in self.session: try: for experiment_name, alternative, goal_name, count in self.session['experiments_goals']: - experiment = experiment_manager.get_experiment(experiment_name) + experiment = experiment_manager.get_experiment( + experiment_name) + if experiment: - self.experiment_counter.increment_goal_count(experiment, alternative, goal_name, self._participant_identifier(), count) + self.experiment_counter.increment_goal_count( + experiment, alternative, goal_name, + self._participant_identifier(), count) except ValueError: pass # Values from older version finally: @@ -403,30 +498,46 @@ def _get_all_enrollments(self): for experiment_name, data in list(enrollments.items()): alternative, _, enrollment_date, last_seen = _session_enrollment_latest_version(data) experiment = experiment_manager.get_experiment(experiment_name) + if experiment: - yield EnrollmentData(experiment, alternative, enrollment_date, last_seen) + yield EnrollmentData( + experiment, alternative, enrollment_date, last_seen) def _cancel_enrollment(self, experiment): alternative = self._get_enrollment(experiment) + if alternative: - self.experiment_counter.remove_participant(experiment, alternative, self._participant_identifier()) + self.experiment_counter.remove_participant( + experiment, alternative, self._participant_identifier()) enrollments = self.session.get('experiments_enrollments', None) + del enrollments[experiment.name] + self.session['experiments_enrollments'] = enrollments def _experiment_goal(self, experiment, alternative, goal_name, count): if self._is_verified_human(): - self.experiment_counter.increment_goal_count(experiment, alternative, goal_name, self._participant_identifier(), count) + self.experiment_counter.increment_goal_count( + experiment, alternative, goal_name, + self._participant_identifier(), count) else: goals = self.session.get('experiments_goals', []) goals.append((experiment.name, alternative, goal_name, count)) self.session['experiments_goals'] = goals - logger.info(json.dumps({'type': 'goal_hit_unconfirmed', 'goal': goal_name, 'goal_count': count, 'experiment': experiment.name, 'alternative': alternative, 'participant': self._participant_identifier()})) + logger.info(json.dumps({ + 'type': 'goal_hit_unconfirmed', + 'goal': goal_name, + 'goal_count': count, + 'experiment': experiment.name, + 'alternative': alternative, + 'participant': self._participant_identifier()})) def _set_last_seen(self, experiment, last_seen): enrollments = self.session.get('experiments_enrollments', {}) alternative, unused, enrollment_date, _ = _session_enrollment_latest_version(enrollments[experiment.name]) - enrollments[experiment.name] = (alternative, unused, timestamp_from_datetime(enrollment_date), timestamp_from_datetime(last_seen)) + enrollments[experiment.name] = ( + alternative, unused, timestamp_from_datetime(enrollment_date), + timestamp_from_datetime(last_seen)) self.session['experiments_enrollments'] = enrollments diff --git a/experiments/views.py b/experiments/views.py index c8bdada0..031daa9f 100644 --- a/experiments/views.py +++ b/experiments/views.py @@ -7,16 +7,17 @@ from experiments.models import Experiment from experiments import conf -TRANSPARENT_1X1_PNG = \ -("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52" - "\x00\x00\x00\x01\x00\x00\x00\x01\x08\x03\x00\x00\x00\x28\xcb\x34" - "\xbb\x00\x00\x00\x19\x74\x45\x58\x74\x53\x6f\x66\x74\x77\x61\x72" - "\x65\x00\x41\x64\x6f\x62\x65\x20\x49\x6d\x61\x67\x65\x52\x65\x61" - "\x64\x79\x71\xc9\x65\x3c\x00\x00\x00\x06\x50\x4c\x54\x45\x00\x00" - "\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01\x74\x52\x4e\x53" - "\x00\x40\xe6\xd8\x66\x00\x00\x00\x0c\x49\x44\x41\x54\x78\xda\x62" - "\x60\x00\x08\x30\x00\x00\x02\x00\x01\x4f\x6d\x59\xe1\x00\x00\x00" - "\x00\x49\x45\x4e\x44\xae\x42\x60\x82\x00") +TRANSPARENT_1X1_PNG = ( + "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52" + "\x00\x00\x00\x01\x00\x00\x00\x01\x08\x03\x00\x00\x00\x28\xcb\x34" + "\xbb\x00\x00\x00\x19\x74\x45\x58\x74\x53\x6f\x66\x74\x77\x61\x72" + "\x65\x00\x41\x64\x6f\x62\x65\x20\x49\x6d\x61\x67\x65\x52\x65\x61" + "\x64\x79\x71\xc9\x65\x3c\x00\x00\x00\x06\x50\x4c\x54\x45\x00\x00" + "\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01\x74\x52\x4e\x53" + "\x00\x40\xe6\xd8\x66\x00\x00\x00\x0c\x49\x44\x41\x54\x78\xda\x62" + "\x60\x00\x08\x30\x00\x00\x02\x00\x01\x4f\x6d\x59\xe1\x00\x00\x00" + "\x00\x49\x45\x4e\x44\xae\x42\x60\x82\x00" +) @never_cache @@ -25,6 +26,7 @@ def confirm_human(request): if conf.CONFIRM_HUMAN: experiment_user = participant(request) experiment_user.confirm_human() + return HttpResponse(status=204)