diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0699f84c..24d95e6a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -65,10 +65,8 @@ jobs: fail-fast: false matrix: python-version: - - "pypy-3.8" - "pypy-3.9" - "pypy-3.10" - - "3.8" - "3.9" - "3.10" - "3.11" @@ -93,12 +91,11 @@ jobs: fail-fast: false matrix: python-version: - - "pypy-3.8" - - "pypy-3.9" - - "pypy-3.10" - - "3.8" - - "3.9" - - "3.10" + # disabled due to librt is not available on this pypy version + # - "pypy-3.9" + # - "pypy-3.10" + # - "3.9" + # - "3.10" - "3.11" - "3.12" steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cd8b71..f5091a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,83 @@ # Optimizely Python SDK Changelog +## 5.4.0 +December 19th, 2025 + +### New Features + +Holdout features: + +- Added support for Holdouts in Feature Experimentation, enabling global traffic exclusion from experiments for statistical confidence validation. ([#469](https://github.com/optimizely/python-sdk/pull/469)) +- Added holdout support for default decision service ([#467](https://github.com/optimizely/python-sdk/pull/467)) +- Project Config updated for Holdout ([#464](https://github.com/optimizely/python-sdk/pull/464)) + +--- + +### Bug Fixes +- Resolved issues with Holdout impression event handling and notification delivery. ([#471](https://github.com/optimizely/python-sdk/pull/471)) + +--- + + +## 5.3.0 +November 13th, 2025 + +This release introduces **Contextual Multi-Armed Bandit (CMAB)** support in the Optimizely Python SDK along with several enhancements, bug fixes, and infrastructure improvements. + +--- + +### New Features + +#### **CMAB Support** + +A major addition in this release is support for **Contextual Multi-Armed Bandit (CMAB)**, enabling adaptive experimentation based on contextual user data. + +The CMAB feature includes: + +- CMAB Client and Service implementations to communicate with CMAB APIs ([#453](https://github.com/optimizely/python-sdk/pull/453), [#455](https://github.com/optimizely/python-sdk/pull/455)) +- Decision Service updates to handle CMAB-based decisioning ([#457](https://github.com/optimizely/python-sdk/pull/457)) +- Impression event updates to include CMAB UUID ([#458](https://github.com/optimizely/python-sdk/pull/458)) +- Cache system enhancements including a new `remove()` method for CMAB cache ([#454](https://github.com/optimizely/python-sdk/pull/454)) +- Configuration options to customize CMAB cache parameters ([#463](https://github.com/optimizely/python-sdk/pull/463)) +- Exposure of CMAB prediction endpoint for programmatic access ([#466](https://github.com/optimizely/python-sdk/pull/466)) + +These updates collectively enable Python SDK users to leverage machine learning–driven bandit optimization strategies within Optimizely Feature Experimentation. + +--- + +#### **Multi-Region Data Hosting** + +- Added SDK support for multi-region data hosting, allowing projects to specify their data residency region ([#459](https://github.com/optimizely/python-sdk/pull/459)). + +--- + +### Security Fixes + +- Addressed a CSRF security warning by properly importing and registering `CSRFProtect` + ([#448](https://github.com/optimizely/python-sdk/pull/448), [#450](https://github.com/optimizely/python-sdk/pull/450), [#452](https://github.com/optimizely/python-sdk/pull/452)). + +--- + +### Enhancements + +- Updated project configuration to track CMAB properties ([#451](https://github.com/optimizely/python-sdk/pull/451)). +- Removed unused `testapp` folder from tests as part of maintenance cleanup ([#461](https://github.com/optimizely/python-sdk/pull/461)). + +--- + +### Bug Fixes + +- Fixed concurrency issues in CMAB service to ensure thread-safe cache and decision logic ([#462](https://github.com/optimizely/python-sdk/pull/462)). + +--- + +### New Contributors + +- [@esrakartalOpt](https://github.com/esrakartalOpt) made their first contribution in [#459](https://github.com/optimizely/python-sdk/pull/459). + +--- + + ## 5.2.0 February 26, 2025 diff --git a/optimizely/bucketer.py b/optimizely/bucketer.py index 1bd7ff52..d1e0b047 100644 --- a/optimizely/bucketer.py +++ b/optimizely/bucketer.py @@ -12,7 +12,7 @@ # limitations under the License. from __future__ import annotations -from typing import Optional, TYPE_CHECKING +from typing import Optional, TYPE_CHECKING, cast import math from sys import version_info @@ -22,13 +22,13 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime from .project_config import ProjectConfig - from .entities import Experiment, Variation + from .entities import Experiment, Variation, Holdout from .helpers.types import TrafficAllocation @@ -104,8 +104,8 @@ def find_bucket( def bucket( self, project_config: ProjectConfig, - experiment: Experiment, user_id: str, bucketing_id: str - ) -> tuple[Optional[Variation], list[str]]: + experiment: Experiment | Holdout, user_id: str, bucketing_id: str + ) -> tuple[Variation | None, list[str]]: """ For a given experiment and bucketing ID determines variation to be shown to user. Args: @@ -119,21 +119,36 @@ def bucket( and array of log messages representing decision making. */. """ - variation_id, decide_reasons = self.bucket_to_entity_id(project_config, experiment, user_id, bucketing_id) - if variation_id: - variation = project_config.get_variation_from_id_by_experiment_id(experiment.id, variation_id) - return variation, decide_reasons + # Check if experiment is None first + if not experiment: + message = 'Invalid entity key provided for bucketing. Returning nil.' + project_config.logger.debug(message) + return None, [] + + # Handle both Experiment and Holdout entities + experiment_key = experiment.key + experiment_id = experiment.id - else: - message = 'Bucketed into an empty traffic range. Returning nil.' - project_config.logger.info(message) - decide_reasons.append(message) + if not experiment_key or not experiment_key.strip(): + message = 'Invalid entity key provided for bucketing. Returning nil.' + project_config.logger.debug(message) + return None, [] + variation_id, decide_reasons = self.bucket_to_entity_id(project_config, experiment, user_id, bucketing_id) + if variation_id: + variation = project_config.get_variation_from_id_by_experiment_id(experiment_id, variation_id) + # Cast is safe here because experiments always use Variation entities, not VariationDict + return cast('Optional[Variation]', variation), decide_reasons + + # No variation found - log message for empty traffic range + message = 'Bucketed into an empty traffic range. Returning nil.' + project_config.logger.info(message) + decide_reasons.append(message) return None, decide_reasons def bucket_to_entity_id( self, project_config: ProjectConfig, - experiment: Experiment, user_id: str, bucketing_id: str + experiment: Experiment | Holdout, user_id: str, bucketing_id: str ) -> tuple[Optional[str], list[str]]: """ For a given experiment and bucketing ID determines variation ID to be shown to user. @@ -151,44 +166,55 @@ def bucket_to_entity_id( if not experiment: return None, decide_reasons - # Determine if experiment is in a mutually exclusive group. - # This will not affect evaluation of rollout rules. - if experiment.groupPolicy in GROUP_POLICIES: - group = project_config.get_group(experiment.groupId) - - if not group: - return None, decide_reasons - - user_experiment_id = self.find_bucket( - project_config, bucketing_id, experiment.groupId, group.trafficAllocation, - ) - - if not user_experiment_id: - message = f'User "{user_id}" is in no experiment.' + # Handle both Experiment and Holdout entities + # Both entities have key, id, and trafficAllocation attributes + from . import entities + + experiment_key = experiment.key + experiment_id = experiment.id + traffic_allocations = experiment.trafficAllocation + + # Determine if experiment is in a mutually exclusive group + # Holdouts don't have groupId or groupPolicy - use isinstance for type narrowing + if isinstance(experiment, entities.Experiment): + group_policy = getattr(experiment, 'groupPolicy', None) + if group_policy and group_policy in GROUP_POLICIES: + group = project_config.get_group(experiment.groupId) + + if not group: + return None, decide_reasons + + user_experiment_id = self.find_bucket( + project_config, bucketing_id, experiment.groupId, group.trafficAllocation, + ) + + if not user_experiment_id: + message = f'User "{user_id}" is in no experiment.' + project_config.logger.info(message) + decide_reasons.append(message) + return None, decide_reasons + + if user_experiment_id != experiment_id: + message = f'User "{user_id}" is not in experiment "{experiment_key}" of group {experiment.groupId}.' + project_config.logger.info(message) + decide_reasons.append(message) + return None, decide_reasons + + message = f'User "{user_id}" is in experiment {experiment_key} of group {experiment.groupId}.' project_config.logger.info(message) decide_reasons.append(message) - return None, decide_reasons - if user_experiment_id != experiment.id: - message = f'User "{user_id}" is not in experiment "{experiment.key}" of group {experiment.groupId}.' - project_config.logger.info(message) - decide_reasons.append(message) - return None, decide_reasons - - message = f'User "{user_id}" is in experiment {experiment.key} of group {experiment.groupId}.' - project_config.logger.info(message) - decide_reasons.append(message) - - traffic_allocations: list[TrafficAllocation] = experiment.trafficAllocation - if experiment.cmab: + # Holdouts don't have cmab - use isinstance for type narrowing + if isinstance(experiment, entities.Experiment) and experiment.cmab: traffic_allocations = [ { "entityId": "$", "endOfRange": experiment.cmab['trafficAllocation'] } ] + # Bucket user if not in white-list and in group (if any) variation_id = self.find_bucket(project_config, bucketing_id, - experiment.id, traffic_allocations) + experiment_id, traffic_allocations) return variation_id, decide_reasons diff --git a/optimizely/cmab/cmab_client.py b/optimizely/cmab/cmab_client.py index dfcffa78..4880b0bb 100644 --- a/optimizely/cmab/cmab_client.py +++ b/optimizely/cmab/cmab_client.py @@ -20,11 +20,12 @@ from optimizely.exceptions import CmabFetchError, CmabInvalidResponseError # Default constants for CMAB requests -DEFAULT_MAX_RETRIES = 3 +DEFAULT_MAX_RETRIES = 1 DEFAULT_INITIAL_BACKOFF = 0.1 # in seconds (100 ms) DEFAULT_MAX_BACKOFF = 10 # in seconds DEFAULT_BACKOFF_MULTIPLIER = 2.0 MAX_WAIT_TIME = 10.0 +DEFAULT_PREDICTION_ENDPOINT = "https://prediction.cmab.optimizely.com/predict/{}" class CmabRetryConfig: @@ -52,17 +53,21 @@ class DefaultCmabClient: """ def __init__(self, http_client: Optional[requests.Session] = None, retry_config: Optional[CmabRetryConfig] = None, - logger: Optional[_logging.Logger] = None): + logger: Optional[_logging.Logger] = None, + prediction_endpoint: Optional[str] = None): """Initialize the CMAB client. Args: http_client (Optional[requests.Session]): HTTP client for making requests. retry_config (Optional[CmabRetryConfig]): Configuration for retry logic. logger (Optional[_logging.Logger]): Logger for logging messages. + prediction_endpoint (Optional[str]): Custom prediction endpoint URL template. + Use {} as placeholder for rule_id. """ self.http_client = http_client or requests.Session() self.retry_config = retry_config self.logger = _logging.adapt_logger(logger or _logging.NoOpLogger()) + self.prediction_endpoint = prediction_endpoint or DEFAULT_PREDICTION_ENDPOINT def fetch_decision( self, @@ -84,7 +89,7 @@ def fetch_decision( Returns: str: The variation ID. """ - url = f"https://prediction.cmab.optimizely.com/predict/{rule_id}" + url = self.prediction_endpoint.format(rule_id) cmab_attributes = [ {"id": key, "value": value, "type": "custom_attribute"} for key, value in attributes.items() diff --git a/optimizely/cmab/cmab_service.py b/optimizely/cmab/cmab_service.py index a7c4b69b..00e996ca 100644 --- a/optimizely/cmab/cmab_service.py +++ b/optimizely/cmab/cmab_service.py @@ -13,14 +13,20 @@ import uuid import json import hashlib +import threading -from typing import Optional, List, TypedDict +from typing import Optional, List, TypedDict, Tuple from optimizely.cmab.cmab_client import DefaultCmabClient from optimizely.odp.lru_cache import LRUCache from optimizely.optimizely_user_context import OptimizelyUserContext, UserAttributes from optimizely.project_config import ProjectConfig from optimizely.decision.optimizely_decide_option import OptimizelyDecideOption from optimizely import logger as _logging +from optimizely.lib import pymmh3 as mmh3 + +NUM_LOCK_STRIPES = 1000 +DEFAULT_CMAB_CACHE_TIMEOUT = 30 * 60 # 30 minutes +DEFAULT_CMAB_CACHE_SIZE = 10000 class CmabDecision(TypedDict): @@ -52,21 +58,50 @@ def __init__(self, cmab_cache: LRUCache[str, CmabCacheValue], self.cmab_cache = cmab_cache self.cmab_client = cmab_client self.logger = logger + self.locks = [threading.Lock() for _ in range(NUM_LOCK_STRIPES)] + + def _get_lock_index(self, user_id: str, rule_id: str) -> int: + """Calculate the lock index for a given user and rule combination.""" + # Create a hash of user_id + rule_id for consistent lock selection + hash_input = f"{user_id}{rule_id}" + hash_value = mmh3.hash(hash_input, seed=0) & 0xFFFFFFFF # Convert to unsigned + return hash_value % NUM_LOCK_STRIPES def get_decision(self, project_config: ProjectConfig, user_context: OptimizelyUserContext, - rule_id: str, options: List[str]) -> CmabDecision: + rule_id: str, options: List[str]) -> Tuple[CmabDecision, List[str]]: + + lock_index = self._get_lock_index(user_context.user_id, rule_id) + with self.locks[lock_index]: + return self._get_decision(project_config, user_context, rule_id, options) + + def _get_decision(self, project_config: ProjectConfig, user_context: OptimizelyUserContext, + rule_id: str, options: List[str]) -> Tuple[CmabDecision, List[str]]: filtered_attributes = self._filter_attributes(project_config, user_context, rule_id) + reasons = [] if OptimizelyDecideOption.IGNORE_CMAB_CACHE in options: - return self._fetch_decision(rule_id, user_context.user_id, filtered_attributes) + reason = f"Ignoring CMAB cache for user '{user_context.user_id}' and rule '{rule_id}'" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) + cmab_decision = self._fetch_decision(rule_id, user_context.user_id, filtered_attributes) + return cmab_decision, reasons if OptimizelyDecideOption.RESET_CMAB_CACHE in options: + reason = f"Resetting CMAB cache for user '{user_context.user_id}' and rule '{rule_id}'" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) self.cmab_cache.reset() cache_key = self._get_cache_key(user_context.user_id, rule_id) if OptimizelyDecideOption.INVALIDATE_USER_CMAB_CACHE in options: + reason = f"Invalidating CMAB cache for user '{user_context.user_id}' and rule '{rule_id}'" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) self.cmab_cache.remove(cache_key) cached_value = self.cmab_cache.lookup(cache_key) @@ -75,17 +110,39 @@ def get_decision(self, project_config: ProjectConfig, user_context: OptimizelyUs if cached_value: if cached_value['attributes_hash'] == attributes_hash: - return CmabDecision(variation_id=cached_value['variation_id'], cmab_uuid=cached_value['cmab_uuid']) + reason = f"CMAB cache hit for user '{user_context.user_id}' and rule '{rule_id}'" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) + return CmabDecision(variation_id=cached_value['variation_id'], + cmab_uuid=cached_value['cmab_uuid']), reasons else: + reason = ( + f"CMAB cache attributes mismatch for user '{user_context.user_id}' " + f"and rule '{rule_id}', fetching new decision." + ) + if self.logger: + self.logger.debug(reason) + reasons.append(reason) self.cmab_cache.remove(cache_key) + else: + reason = f"CMAB cache miss for user '{user_context.user_id}' and rule '{rule_id}'" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) cmab_decision = self._fetch_decision(rule_id, user_context.user_id, filtered_attributes) + reason = f"CMAB decision is {cmab_decision}" + if self.logger: + self.logger.debug(reason) + reasons.append(reason) + self.cmab_cache.save(cache_key, { 'attributes_hash': attributes_hash, 'variation_id': cmab_decision['variation_id'], 'cmab_uuid': cmab_decision['cmab_uuid'], }) - return cmab_decision + return cmab_decision, reasons def _fetch_decision(self, rule_id: str, user_id: str, attributes: UserAttributes) -> CmabDecision: cmab_uuid = str(uuid.uuid4()) diff --git a/optimizely/decision/optimizely_decide_option.py b/optimizely/decision/optimizely_decide_option.py index 8cffcfec..0443ddef 100644 --- a/optimizely/decision/optimizely_decide_option.py +++ b/optimizely/decision/optimizely_decide_option.py @@ -16,7 +16,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final class OptimizelyDecideOption: diff --git a/optimizely/decision/optimizely_decision_message.py b/optimizely/decision/optimizely_decision_message.py index 20231ea5..c6178322 100644 --- a/optimizely/decision/optimizely_decision_message.py +++ b/optimizely/decision/optimizely_decision_message.py @@ -16,7 +16,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final class OptimizelyDecisionMessage: diff --git a/optimizely/decision_service.py b/optimizely/decision_service.py index d22bec87..be2be2c5 100644 --- a/optimizely/decision_service.py +++ b/optimizely/decision_service.py @@ -12,7 +12,9 @@ # limitations under the License. from __future__ import annotations -from typing import TYPE_CHECKING, NamedTuple, Optional, Sequence, List, TypedDict +from typing import TYPE_CHECKING, NamedTuple, Optional, Sequence, List, TypedDict, Union + +from optimizely.helpers.types import VariationDict from . import bucketer from . import entities @@ -59,7 +61,7 @@ class VariationResult(TypedDict): cmab_uuid: Optional[str] error: bool reasons: List[str] - variation: Optional[entities.Variation] + variation: Optional[Union[entities.Variation, VariationDict]] class DecisionResult(TypedDict): @@ -79,8 +81,8 @@ class DecisionResult(TypedDict): class Decision(NamedTuple): """Named tuple containing selected experiment, variation, source and cmab_uuid. None if no experiment/variation was selected.""" - experiment: Optional[entities.Experiment] - variation: Optional[entities.Variation] + experiment: Optional[Union[entities.Experiment, entities.Holdout]] + variation: Optional[Union[entities.Variation, VariationDict]] source: Optional[str] cmab_uuid: Optional[str] @@ -175,9 +177,10 @@ def _get_decision_for_cmab_experiment( # User is in CMAB allocation, proceed to CMAB decision try: options_list = list(options) if options is not None else [] - cmab_decision = self.cmab_service.get_decision( + cmab_decision, cmab_reasons = self.cmab_service.get_decision( project_config, user_context, experiment.id, options_list ) + decide_reasons.extend(cmab_reasons) return { "error": False, "result": cmab_decision, @@ -243,7 +246,11 @@ def set_forced_variation( # The invalid variation key will be logged inside this call. return False - variation_id = forced_variation.id + # Handle both Variation entity and VariationDict + if isinstance(forced_variation, dict): + variation_id = forced_variation['id'] + else: + variation_id = forced_variation.id if user_id not in self.forced_variation_map: self.forced_variation_map[user_id] = {experiment_id: variation_id} @@ -258,7 +265,7 @@ def set_forced_variation( def get_forced_variation( self, project_config: ProjectConfig, experiment_key: str, user_id: str - ) -> tuple[Optional[entities.Variation], list[str]]: + ) -> tuple[Optional[Union[entities.Variation, VariationDict]], list[str]]: """ Gets the forced variation key for the given user and experiment. Args: @@ -299,7 +306,9 @@ def get_forced_variation( if variation is None: return None, decide_reasons - message = f'Variation "{variation.key}" is mapped to experiment "{experiment_key}" and ' \ + # Handle both Variation entity and VariationDict + var_key = variation['key'] if isinstance(variation, dict) else variation.key + message = f'Variation "{var_key}" is mapped to experiment "{experiment_key}" and ' \ f'user "{user_id}" in the forced variation map' self.logger.debug(message) decide_reasons.append(message) @@ -307,7 +316,7 @@ def get_forced_variation( def get_whitelisted_variation( self, project_config: ProjectConfig, experiment: entities.Experiment, user_id: str - ) -> tuple[Optional[entities.Variation], list[str]]: + ) -> tuple[Optional[Union[entities.Variation, VariationDict]], list[str]]: """ Determine if a user is forced into a variation (through whitelisting) for the given experiment and return that variation. @@ -338,7 +347,7 @@ def get_whitelisted_variation( def get_stored_variation( self, project_config: ProjectConfig, experiment: entities.Experiment, user_profile: UserProfile - ) -> Optional[entities.Variation]: + ) -> Optional[Union[entities.Variation, VariationDict]]: """ Determine if the user has a stored variation available for the given experiment and return that. Args: @@ -355,8 +364,10 @@ def get_stored_variation( if variation_id: variation = project_config.get_variation_from_id(experiment.key, variation_id) if variation: + # Handle both Variation entity and VariationDict + var_key = variation['key'] if isinstance(variation, dict) else variation.key message = f'Found a stored decision. User "{user_id}" is in ' \ - f'variation "{variation.key}" of experiment "{experiment.key}".' + f'variation "{var_key}" of experiment "{experiment.key}".' self.logger.info(message) return variation @@ -423,7 +434,7 @@ def get_variation( } # Check if the user is forced into a variation - variation: Optional[entities.Variation] + variation: Optional[Union[entities.Variation, VariationDict]] variation, reasons_received = self.get_forced_variation(project_config, experiment.key, user_id) decide_reasons += reasons_received if variation: @@ -504,6 +515,11 @@ def get_variation( 'reasons': decide_reasons, 'variation': None } + ignore_user_profile = True + self.logger.debug( + f'Skipping user profile service for CMAB experiment "{experiment.key}". ' + f'CMAB decisions are dynamic and not stored for sticky bucketing.' + ) variation_id = cmab_decision['variation_id'] if cmab_decision else None cmab_uuid = cmab_decision['cmab_uuid'] if cmab_decision else None variation = project_config.get_variation_from_id(experiment_key=experiment.key, @@ -669,7 +685,247 @@ def get_variation_for_feature( - 'error': Boolean indicating if an error occurred during the decision process. - 'reasons': List of log messages representing decision making for the feature. """ - return self.get_variations_for_feature_list(project_config, [feature], user_context, options)[0] + # Check if user profile service should be ignored + if options: + ignore_ups = OptimizelyDecideOption.IGNORE_USER_PROFILE_SERVICE in options + else: + ignore_ups = False + + # Create user profile tracker for sticky bucketing + user_profile_tracker: Optional[UserProfileTracker] = None + if self.user_profile_service is not None and not ignore_ups: + user_profile_tracker = UserProfileTracker(user_context.user_id, self.user_profile_service, self.logger) + # Load user profile once before processing + user_profile_tracker.load_user_profile([], None) + + result = self.get_decision_for_flag(feature, user_context, project_config, options, user_profile_tracker) + + # Save user profile after decision + if user_profile_tracker is not None and not ignore_ups: + user_profile_tracker.save_user_profile() + + return result + + def get_decision_for_flag( + self, + feature_flag: entities.FeatureFlag, + user_context: OptimizelyUserContext, + project_config: ProjectConfig, + decide_options: Optional[Sequence[str]] = None, + user_profile_tracker: Optional[UserProfileTracker] = None, + decide_reasons: Optional[list[str]] = None + ) -> DecisionResult: + """ + Get the decision for a single feature flag. + Processes holdouts, experiments, and rollouts in that order. + + Args: + feature_flag: The feature flag to get a decision for. + user_context: The user context. + project_config: The project config. + decide_options: Sequence of decide options. + user_profile_tracker: The user profile tracker. + decide_reasons: List of decision reasons to merge. + + Returns: + A DecisionResult for the feature flag. + """ + reasons = decide_reasons.copy() if decide_reasons else [] + user_id = user_context.user_id + + # Check holdouts + holdouts = project_config.get_holdouts_for_flag(feature_flag.key) + for holdout in holdouts: + holdout_decision = self.get_variation_for_holdout(holdout, user_context, project_config) + reasons.extend(holdout_decision['reasons']) + + decision = holdout_decision['decision'] + # Check if user was bucketed into holdout (has a variation) + if decision.variation is None: + continue + + message = ( + f"The user '{user_id}' is bucketed into holdout '{holdout.key}' " + f"for feature flag '{feature_flag.key}'." + ) + self.logger.info(message) + reasons.append(message) + return { + 'decision': holdout_decision['decision'], + 'error': False, + 'reasons': reasons + } + + # If no holdout decision, check experiments then rollouts + if feature_flag.experimentIds: + for experiment_id in feature_flag.experimentIds: + experiment = project_config.get_experiment_from_id(experiment_id) + + if experiment: + # Check for forced decision + optimizely_decision_context = OptimizelyUserContext.OptimizelyDecisionContext( + feature_flag.key, experiment.key) + forced_decision_variation, forced_reasons = self.validated_forced_decision( + project_config, optimizely_decision_context, user_context) + reasons.extend(forced_reasons) + + if forced_decision_variation: + decision = Decision(experiment, forced_decision_variation, + enums.DecisionSources.FEATURE_TEST, None) + return { + 'decision': decision, + 'error': False, + 'reasons': reasons + } + + # Get variation for experiment + variation_result = self.get_variation( + project_config, experiment, user_context, user_profile_tracker, reasons, decide_options + ) + reasons.extend(variation_result['reasons']) + + if variation_result['error']: + decision = Decision(experiment, None, enums.DecisionSources.FEATURE_TEST, + variation_result['cmab_uuid']) + return { + 'decision': decision, + 'error': True, + 'reasons': reasons + } + + if variation_result['variation']: + decision = Decision(experiment, variation_result['variation'], + enums.DecisionSources.FEATURE_TEST, + variation_result['cmab_uuid']) + return { + 'decision': decision, + 'error': False, + 'reasons': reasons + } + + # If no experiment decision, check rollouts + rollout_decision, rollout_reasons = self.get_variation_for_rollout( + project_config, feature_flag, user_context + ) + if rollout_reasons: + reasons.extend(rollout_reasons) + + # Log rollout decision for backward compatibility with tests + has_variation = False + if isinstance(rollout_decision, Decision): + has_variation = rollout_decision.variation is not None + else: + # Handle mocked return values in tests + has_variation = rollout_decision is not None + + if has_variation: + self.logger.debug(f'User "{user_id}" bucketed into rollout for feature "{feature_flag.key}".') + else: + self.logger.debug(f'User "{user_id}" not bucketed into any rollout for feature "{feature_flag.key}".') + + return { + 'decision': rollout_decision, + 'error': False, + 'reasons': reasons + } + + def get_variation_for_holdout( + self, + holdout: entities.Holdout, + user_context: OptimizelyUserContext, + project_config: ProjectConfig + ) -> DecisionResult: + """ + Get the variation for holdout. + + Args: + holdout: The holdout configuration (Holdout entity). + user_context: The user context. + project_config: The project config. + + Returns: + A DecisionResult for the holdout. + """ + from optimizely.helpers.enums import ExperimentAudienceEvaluationLogs + + decide_reasons: list[str] = [] + user_id = user_context.user_id + attributes = user_context.get_user_attributes() + + # Check if holdout is activated (Running status) + if not holdout.is_activated: + message = f"Holdout '{holdout.key}' is not running." + self.logger.info(message) + decide_reasons.append(message) + return { + 'decision': Decision(None, None, enums.DecisionSources.HOLDOUT, None), + 'error': False, + 'reasons': decide_reasons + } + + bucketing_id, bucketing_id_reasons = self._get_bucketing_id(user_id, attributes) + decide_reasons.extend(bucketing_id_reasons) + + # Check audience conditions using the same method as experiments + audience_conditions = holdout.get_audience_conditions_or_ids() + user_meets_audience_conditions, reasons_received = audience_helper.does_user_meet_audience_conditions( + project_config, + audience_conditions, + ExperimentAudienceEvaluationLogs, + holdout.key, + user_context, + self.logger + ) + decide_reasons.extend(reasons_received) + + if not user_meets_audience_conditions: + message = ( + f"User '{user_id}' does not meet the conditions for holdout " + f"'{holdout.key}'." + ) + self.logger.debug(message) + decide_reasons.append(message) + return { + 'decision': Decision(None, None, enums.DecisionSources.HOLDOUT, None), + 'error': False, + 'reasons': decide_reasons + } + + # Bucket user into holdout variation + variation, bucket_reasons = self.bucketer.bucket( + project_config, holdout, user_id, bucketing_id + ) + decide_reasons.extend(bucket_reasons) + + if variation: + variation_key = variation.get('key') if isinstance(variation, dict) else variation.key + message = ( + f"The user '{user_id}' is bucketed into variation '{variation_key}' " + f"of holdout '{holdout.key}'." + ) + self.logger.info(message) + decide_reasons.append(message) + + holdout_decision: Decision = Decision( + experiment=holdout, + variation=variation, + source=enums.DecisionSources.HOLDOUT, + cmab_uuid=None + ) + return { + 'decision': holdout_decision, + 'error': False, + 'reasons': decide_reasons + } + + message = f"User '{user_id}' is not bucketed into any variation for holdout '{holdout.key}'." + self.logger.info(message) + decide_reasons.append(message) + return { + 'decision': Decision(None, None, enums.DecisionSources.HOLDOUT, None), + 'error': False, + 'reasons': decide_reasons + } def validated_forced_decision( self, @@ -756,100 +1012,37 @@ def get_variations_for_feature_list( - 'error': Boolean indicating if an error occurred during the decision process. - 'reasons': List of log messages representing decision making for each feature. """ - decide_reasons: list[str] = [] + user_id = user_context.user_id + # Check if user profile service should be ignored if options: ignore_ups = OptimizelyDecideOption.IGNORE_USER_PROFILE_SERVICE in options else: ignore_ups = False + # Create user profile tracker once for all features user_profile_tracker: Optional[UserProfileTracker] = None if self.user_profile_service is not None and not ignore_ups: - user_profile_tracker = UserProfileTracker(user_context.user_id, self.user_profile_service, self.logger) - user_profile_tracker.load_user_profile(decide_reasons, None) + user_profile_tracker = UserProfileTracker(user_id, self.user_profile_service, self.logger) + # Load user profile once before processing features + user_profile_tracker.load_user_profile([], None) - decisions = [] + # Process each feature by delegating to get_decision_for_flag + decisions: list[DecisionResult] = [] for feature in features: - feature_reasons = decide_reasons.copy() - experiment_decision_found = False # Track if an experiment decision was made for the feature - - # Check if the feature flag is under an experiment - if feature.experimentIds: - for experiment_id in feature.experimentIds: - experiment = project_config.get_experiment_from_id(experiment_id) - decision_variation = None - - if experiment: - optimizely_decision_context = OptimizelyUserContext.OptimizelyDecisionContext( - feature.key, experiment.key) - forced_decision_variation, reasons_received = self.validated_forced_decision( - project_config, optimizely_decision_context, user_context) - feature_reasons.extend(reasons_received) - - if forced_decision_variation: - decision_variation = forced_decision_variation - cmab_uuid = None - error = False - else: - variation_result = self.get_variation( - project_config, experiment, user_context, user_profile_tracker, feature_reasons, options - ) - cmab_uuid = variation_result['cmab_uuid'] - variation_reasons = variation_result['reasons'] - decision_variation = variation_result['variation'] - error = variation_result['error'] - feature_reasons.extend(variation_reasons) - - if error: - decision = Decision(experiment, None, enums.DecisionSources.FEATURE_TEST, cmab_uuid) - decision_result: DecisionResult = { - 'decision': decision, - 'error': True, - 'reasons': feature_reasons - } - decisions.append(decision_result) - experiment_decision_found = True - break - - if decision_variation: - self.logger.debug( - f'User "{user_context.user_id}" ' - f'bucketed into experiment "{experiment.key}" of feature "{feature.key}".' - ) - decision = Decision(experiment, decision_variation, - enums.DecisionSources.FEATURE_TEST, cmab_uuid) - decision_result = { - 'decision': decision, - 'error': False, - 'reasons': feature_reasons - } - decisions.append(decision_result) - experiment_decision_found = True # Mark that a decision was found - break # Stop after the first successful experiment decision - - # Only process rollout if no experiment decision was found and no error - if not experiment_decision_found: - rollout_decision, rollout_reasons = self.get_variation_for_rollout(project_config, - feature, - user_context) - if rollout_reasons: - feature_reasons.extend(rollout_reasons) - if rollout_decision: - self.logger.debug(f'User "{user_context.user_id}" ' - f'bucketed into rollout for feature "{feature.key}".') - else: - self.logger.debug(f'User "{user_context.user_id}" ' - f'not bucketed into any rollout for feature "{feature.key}".') - - decision_result = { - 'decision': rollout_decision, - 'error': False, - 'reasons': feature_reasons - } - decisions.append(decision_result) + flag_decision_result = self.get_decision_for_flag( + feature_flag=feature, + user_context=user_context, + project_config=project_config, + decide_options=options, + user_profile_tracker=user_profile_tracker, + decide_reasons=None + ) + decisions.append(flag_decision_result) - if self.user_profile_service is not None and user_profile_tracker is not None and ignore_ups is False: + # Save user profile once after all features processed + if self.user_profile_service is not None and user_profile_tracker is not None and not ignore_ups: user_profile_tracker.save_user_profile() return decisions diff --git a/optimizely/entities.py b/optimizely/entities.py index 7d257656..12f4f849 100644 --- a/optimizely/entities.py +++ b/optimizely/entities.py @@ -17,7 +17,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: @@ -27,6 +27,8 @@ class BaseEntity: def __eq__(self, other: object) -> bool: + if not hasattr(other, '__dict__'): + return False return self.__dict__ == other.__dict__ @@ -194,3 +196,76 @@ def __init__(self, key: str, host: Optional[str] = None, publicKey: Optional[str self.key = key self.host = host self.publicKey = publicKey + + +class Holdout(BaseEntity): + """Holdout entity representing a holdout experiment for measuring incremental impact. + + Aligned with Swift SDK implementation where Holdout is a proper entity class + that conforms to ExperimentCore protocol. + """ + + class Status: + """Holdout status constants matching Swift enum Status.""" + DRAFT: Final = 'Draft' + RUNNING: Final = 'Running' + CONCLUDED: Final = 'Concluded' + ARCHIVED: Final = 'Archived' + + def __init__( + self, + id: str, + key: str, + status: str, + variations: list[VariationDict], + trafficAllocation: list[TrafficAllocation], + audienceIds: list[str], + includedFlags: Optional[list[str]] = None, + excludedFlags: Optional[list[str]] = None, + audienceConditions: Optional[Sequence[str | list[str]]] = None, + **kwargs: Any + ): + self.id = id + self.key = key + self.status = status + self.variations = variations + self.trafficAllocation = trafficAllocation + self.audienceIds = audienceIds + self.audienceConditions = audienceConditions + self.includedFlags = includedFlags or [] + self.excludedFlags = excludedFlags or [] + + def get_audience_conditions_or_ids(self) -> Sequence[str | list[str]]: + """Returns audienceConditions if present, otherwise audienceIds. + + This matches the Experiment.get_audience_conditions_or_ids() method + and enables holdouts to work with the same audience evaluation logic. + """ + return self.audienceConditions if self.audienceConditions is not None else self.audienceIds + + @property + def is_activated(self) -> bool: + """Check if the holdout is activated (running). + + Matches Swift's isActivated computed property: + var isActivated: Bool { return status == .running } + + Returns: + True if status is 'Running', False otherwise. + """ + return self.status == self.Status.RUNNING + + def __str__(self) -> str: + return self.key + + def __getitem__(self, key: str) -> Any: + """Enable dict-style access for backward compatibility with tests.""" + return getattr(self, key) + + def __setitem__(self, key: str, value: Any) -> None: + """Enable dict-style assignment for backward compatibility with tests.""" + setattr(self, key, value) + + def get(self, key: str, default: Any = None) -> Any: + """Enable dict-style .get() method for backward compatibility with tests.""" + return getattr(self, key, default) diff --git a/optimizely/event/event_factory.py b/optimizely/event/event_factory.py index c872fb17..f66c1d59 100644 --- a/optimizely/event/event_factory.py +++ b/optimizely/event/event_factory.py @@ -25,7 +25,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime @@ -125,10 +125,14 @@ def _create_visitor(cls, event: Optional[user_event.UserEvent], logger: Logger) if isinstance(event.variation, entities.Variation): variation_id = event.variation.id variation_key = event.variation.key + elif isinstance(event.variation, dict): + variation_id = event.variation.get('id', '') + variation_key = event.variation.get('key', '') if event.experiment: - experiment_layerId = event.experiment.layerId experiment_id = event.experiment.id + if isinstance(event.experiment, entities.Experiment): + experiment_layerId = event.experiment.layerId metadata = payload.Metadata(event.flag_key, event.rule_key, event.rule_type, variation_key, diff --git a/optimizely/event/event_processor.py b/optimizely/event/event_processor.py index 05f5e078..bdce6d9d 100644 --- a/optimizely/event/event_processor.py +++ b/optimizely/event/event_processor.py @@ -34,7 +34,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final class BaseEventProcessor(ABC): @@ -72,7 +72,7 @@ class Signal: def __init__( self, - event_dispatcher: Optional[type[EventDispatcher] | CustomEventDispatcher] = None, + event_dispatcher: Optional[EventDispatcher | CustomEventDispatcher] = None, logger: Optional[_logging.Logger] = None, start_on_init: bool = False, event_queue: Optional[queue.Queue[UserEvent | Signal]] = None, diff --git a/optimizely/event/log_event.py b/optimizely/event/log_event.py index 7c0beeb6..49c344dd 100644 --- a/optimizely/event/log_event.py +++ b/optimizely/event/log_event.py @@ -20,7 +20,7 @@ if version_info < (3, 8): from typing_extensions import Literal else: - from typing import Literal # type: ignore + from typing import Literal class LogEvent(event_builder.Event): diff --git a/optimizely/event/user_event.py b/optimizely/event/user_event.py index e257647c..8006c052 100644 --- a/optimizely/event/user_event.py +++ b/optimizely/event/user_event.py @@ -22,14 +22,15 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime - from optimizely.entities import Experiment, Variation, Event + from optimizely.entities import Experiment, Variation, Event, Holdout from optimizely.event.payload import VisitorAttribute from optimizely.helpers.event_tag_utils import EventTags + from optimizely.helpers.types import VariationDict CLIENT_NAME: Final = 'python-sdk' @@ -63,9 +64,9 @@ def __init__( self, event_context: EventContext, user_id: str, - experiment: Experiment, + experiment: Experiment | Holdout, visitor_attributes: list[VisitorAttribute], - variation: Optional[Variation], + variation: Optional[Variation | VariationDict], flag_key: str, rule_key: str, rule_type: str, diff --git a/optimizely/event/user_event_factory.py b/optimizely/event/user_event_factory.py index 7a77720f..a213a278 100644 --- a/optimizely/event/user_event_factory.py +++ b/optimizely/event/user_event_factory.py @@ -23,7 +23,8 @@ # prevent circular dependenacy by skipping import at runtime from optimizely.optimizely_user_context import UserAttributes from optimizely.project_config import ProjectConfig - from optimizely.entities import Experiment, Variation + from optimizely.entities import Experiment, Variation, Holdout + from optimizely.helpers.types import VariationDict class UserEventFactory: @@ -33,7 +34,7 @@ class UserEventFactory: def create_impression_event( cls, project_config: ProjectConfig, - activated_experiment: Experiment, + activated_experiment: Experiment | Holdout, variation_id: Optional[str], flag_key: str, rule_key: str, @@ -64,16 +65,16 @@ def create_impression_event( if not activated_experiment and rule_type is not enums.DecisionSources.ROLLOUT: return None - variation: Optional[Variation] = None + variation: Optional[Variation | VariationDict] = None experiment_id = None if activated_experiment: experiment_id = activated_experiment.id - if variation_id and flag_key: + if variation_id and flag_key and rule_type != enums.DecisionSources.HOLDOUT: # need this condition when we send events involving forced decisions - # (F-to-D or E-to-D with any ruleKey/variationKey combinations) variation = project_config.get_flag_variation(flag_key, 'id', variation_id) elif variation_id and experiment_id: + # For holdouts, experiments, and rollouts - lookup by experiment/holdout ID variation = project_config.get_variation_from_id_by_experiment_id(experiment_id, variation_id) event_context = user_event.EventContext( diff --git a/optimizely/event_builder.py b/optimizely/event_builder.py index 90678830..e9c9fd44 100644 --- a/optimizely/event_builder.py +++ b/optimizely/event_builder.py @@ -25,7 +25,7 @@ if version_info < (3, 8): from typing_extensions import Final, Literal else: - from typing import Final, Literal # type: ignore + from typing import Final, Literal if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime diff --git a/optimizely/event_dispatcher.py b/optimizely/event_dispatcher.py index 767fbb7d..55209dc8 100644 --- a/optimizely/event_dispatcher.py +++ b/optimizely/event_dispatcher.py @@ -26,7 +26,7 @@ if version_info < (3, 8): from typing_extensions import Protocol else: - from typing import Protocol # type: ignore + from typing import Protocol class CustomEventDispatcher(Protocol): diff --git a/optimizely/helpers/condition.py b/optimizely/helpers/condition.py index 58000a90..40338b40 100644 --- a/optimizely/helpers/condition.py +++ b/optimizely/helpers/condition.py @@ -32,7 +32,7 @@ if version_info < (3, 8): from typing_extensions import Literal, Final else: - from typing import Literal, Final # type: ignore + from typing import Literal, Final class ConditionOperatorTypes: diff --git a/optimizely/helpers/enums.py b/optimizely/helpers/enums.py index e3acafef..74acdcfa 100644 --- a/optimizely/helpers/enums.py +++ b/optimizely/helpers/enums.py @@ -17,7 +17,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final class CommonAudienceEvaluationLogs: @@ -99,6 +99,7 @@ class DecisionSources: EXPERIMENT: Final = 'experiment' FEATURE_TEST: Final = 'feature-test' ROLLOUT: Final = 'rollout' + HOLDOUT: Final = 'holdout' class Errors: diff --git a/optimizely/helpers/event_tag_utils.py b/optimizely/helpers/event_tag_utils.py index cb577950..ad90cc13 100644 --- a/optimizely/helpers/event_tag_utils.py +++ b/optimizely/helpers/event_tag_utils.py @@ -21,7 +21,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: diff --git a/optimizely/helpers/sdk_settings.py b/optimizely/helpers/sdk_settings.py index 6b31ee9c..e5e7aeb1 100644 --- a/optimizely/helpers/sdk_settings.py +++ b/optimizely/helpers/sdk_settings.py @@ -33,7 +33,8 @@ def __init__( odp_event_manager: Optional[OdpEventManager] = None, odp_segment_request_timeout: Optional[int] = None, odp_event_request_timeout: Optional[int] = None, - odp_event_flush_interval: Optional[int] = None + odp_event_flush_interval: Optional[int] = None, + cmab_prediction_endpoint: Optional[str] = None ) -> None: """ Args: @@ -52,6 +53,8 @@ def __init__( send successfully (optional). odp_event_request_timeout: Time to wait in seconds for send_odp_events request to send successfully. odp_event_flush_interval: Time to wait for events to accumulate before sending a batch in seconds (optional). + cmab_prediction_endpoint: Custom CMAB prediction endpoint URL template (optional). + Use {} as placeholder for rule_id. Defaults to production endpoint if not provided. """ self.odp_disabled = odp_disabled @@ -63,3 +66,4 @@ def __init__( self.fetch_segments_timeout = odp_segment_request_timeout self.odp_event_timeout = odp_event_request_timeout self.odp_flush_interval = odp_event_flush_interval + self.cmab_prediction_endpoint = cmab_prediction_endpoint diff --git a/optimizely/helpers/types.py b/optimizely/helpers/types.py index 3cca45de..d4177dc0 100644 --- a/optimizely/helpers/types.py +++ b/optimizely/helpers/types.py @@ -12,7 +12,7 @@ # limitations under the License. from __future__ import annotations -from typing import Optional, Any +from typing import Literal, Optional, Any from sys import version_info @@ -115,3 +115,16 @@ class CmabDict(BaseEntity): """Cmab dict from parsed datafile json.""" attributeIds: list[str] trafficAllocation: int + + +HoldoutStatus = Literal['Draft', 'Running', 'Concluded', 'Archived'] + + +class HoldoutDict(ExperimentDict): + """Holdout dict from parsed datafile json. + + Extends ExperimentDict with holdout-specific properties. + """ + holdoutStatus: HoldoutStatus + includedFlags: list[str] + excludedFlags: list[str] diff --git a/optimizely/logger.py b/optimizely/logger.py index 33d3660c..42f879de 100644 --- a/optimizely/logger.py +++ b/optimizely/logger.py @@ -20,7 +20,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final _DEFAULT_LOG_FORMAT: Final = '%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s:%(message)s' diff --git a/optimizely/notification_center.py b/optimizely/notification_center.py index 322a5862..3d0b0cba 100644 --- a/optimizely/notification_center.py +++ b/optimizely/notification_center.py @@ -20,7 +20,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final NOTIFICATION_TYPES: Final = tuple( diff --git a/optimizely/odp/lru_cache.py b/optimizely/odp/lru_cache.py index 073973e6..64337dad 100644 --- a/optimizely/odp/lru_cache.py +++ b/optimizely/odp/lru_cache.py @@ -22,7 +22,7 @@ if version_info < (3, 8): from typing_extensions import Protocol else: - from typing import Protocol # type: ignore + from typing import Protocol # generic type definitions for LRUCache parameters K = TypeVar('K', bound=Hashable, contravariant=True) diff --git a/optimizely/odp/optimizely_odp_option.py b/optimizely/odp/optimizely_odp_option.py index ce6eaf00..94e1e90e 100644 --- a/optimizely/odp/optimizely_odp_option.py +++ b/optimizely/odp/optimizely_odp_option.py @@ -16,7 +16,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final class OptimizelyOdpOption: diff --git a/optimizely/optimizely.py b/optimizely/optimizely.py index ae433cb1..4b014e7f 100644 --- a/optimizely/optimizely.py +++ b/optimizely/optimizely.py @@ -13,7 +13,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Optional, Union + +from optimizely.helpers.types import VariationDict from . import decision_service @@ -46,17 +48,13 @@ from .optimizely_user_context import OptimizelyUserContext, UserAttributes from .project_config import ProjectConfig from .cmab.cmab_client import DefaultCmabClient, CmabRetryConfig -from .cmab.cmab_service import DefaultCmabService, CmabCacheValue +from .cmab.cmab_service import DefaultCmabService, CmabCacheValue, DEFAULT_CMAB_CACHE_SIZE, DEFAULT_CMAB_CACHE_TIMEOUT if TYPE_CHECKING: # prevent circular dependency by skipping import at runtime from .user_profile import UserProfileService from .helpers.event_tag_utils import EventTags -# Default constants for CMAB cache -DEFAULT_CMAB_CACHE_TIMEOUT = 30 * 60 * 1000 # 30 minutes in milliseconds -DEFAULT_CMAB_CACHE_SIZE = 1000 - class Optimizely: """ Class encapsulating all SDK functionality. """ @@ -77,6 +75,7 @@ def __init__( default_decide_options: Optional[list[str]] = None, event_processor_options: Optional[dict[str, Any]] = None, settings: Optional[OptimizelySdkSettings] = None, + cmab_service: Optional[DefaultCmabService] = None, ) -> None: """ Optimizely init method for managing Custom projects. @@ -178,19 +177,90 @@ def __init__( self.event_builder = event_builder.EventBuilder() # Initialize CMAB components - self.cmab_client = DefaultCmabClient( - retry_config=CmabRetryConfig(), - logger=self.logger - ) - self.cmab_cache: LRUCache[str, CmabCacheValue] = LRUCache(DEFAULT_CMAB_CACHE_SIZE, DEFAULT_CMAB_CACHE_TIMEOUT) - self.cmab_service = DefaultCmabService( - cmab_cache=self.cmab_cache, - cmab_client=self.cmab_client, - logger=self.logger - ) + if cmab_service: + self.cmab_service = cmab_service + else: + # Get custom prediction endpoint from settings if provided + cmab_prediction_endpoint = None + if self.sdk_settings and self.sdk_settings.cmab_prediction_endpoint: + cmab_prediction_endpoint = self.sdk_settings.cmab_prediction_endpoint + + self.cmab_client = DefaultCmabClient( + retry_config=CmabRetryConfig(), + logger=self.logger, + prediction_endpoint=cmab_prediction_endpoint + ) + self.cmab_cache: LRUCache[str, CmabCacheValue] = LRUCache(DEFAULT_CMAB_CACHE_SIZE, + DEFAULT_CMAB_CACHE_TIMEOUT) + self.cmab_service = DefaultCmabService( + cmab_cache=self.cmab_cache, + cmab_client=self.cmab_client, + logger=self.logger + ) self.decision_service = decision_service.DecisionService(self.logger, user_profile_service, self.cmab_service) self.user_profile_service = user_profile_service + def _get_variation_key(self, variation: Optional[Union[entities.Variation, VariationDict]]) -> Optional[str]: + """Helper to extract variation key from either dict (holdout) or Variation object. + Args: + variation: Either a dict (from holdout) or entities.Variation object + Returns: + The variation key as a string, or None if not available + """ + if variation is None: + return None + + try: + # Try dict access first (for holdouts) + if isinstance(variation, dict): + return variation.get('key') + # Otherwise assume it's a Variation entity object + else: + return variation.key + except (AttributeError, KeyError, TypeError): + self.logger.warning(f"Unable to extract variation key from {type(variation)}") + return None + + def _get_variation_id(self, variation: Optional[Union[entities.Variation, VariationDict]]) -> Optional[str]: + """Helper to extract variation id from either dict (holdout) or Variation object. + Args: + variation: Either a dict (from holdout) or entities.Variation object + Returns: + The variation id as a string, or None if not available + """ + if variation is None: + return None + + try: + # Try dict access first (for holdouts) + if isinstance(variation, dict): + return variation.get('id') + # Otherwise assume it's a Variation entity object + else: + return variation.id + except (AttributeError, KeyError, TypeError): + self.logger.warning(f"Unable to extract variation id from {type(variation)}") + return None + + def _get_feature_enabled(self, variation: Optional[Union[entities.Variation, VariationDict]]) -> bool: + """Helper to extract featureEnabled flag from either dict (holdout) or Variation object. + Args: + variation: Either a dict (from holdout) or entities.Variation object + Returns: + The featureEnabled value, defaults to False if not available + """ + if variation is None: + return False + + try: + if isinstance(variation, dict): + feature_enabled = variation.get('featureEnabled', False) + return bool(feature_enabled) if feature_enabled is not None else False + else: + return variation.featureEnabled if hasattr(variation, 'featureEnabled') else False + except (AttributeError, KeyError, TypeError): + return False + def _validate_instantiation_options(self) -> None: """ Helper method to validate all instantiation parameters. @@ -259,8 +329,9 @@ def _validate_user_inputs( return True def _send_impression_event( - self, project_config: project_config.ProjectConfig, experiment: Optional[entities.Experiment], - variation: Optional[entities.Variation], flag_key: str, rule_key: str, rule_type: str, + self, project_config: project_config.ProjectConfig, + experiment: Optional[Union[entities.Experiment, entities.Holdout]], + variation: Optional[Union[entities.Variation, VariationDict]], flag_key: str, rule_key: str, rule_type: str, enabled: bool, user_id: str, attributes: Optional[UserAttributes], cmab_uuid: Optional[str] = None ) -> None: """ Helper method to send impression event. @@ -279,7 +350,7 @@ def _send_impression_event( if not experiment: experiment = entities.Experiment.get_default() - variation_id = variation.id if variation is not None else None + variation_id = self._get_variation_id(variation) if variation is not None else None user_event = user_event_factory.UserEventFactory.create_impression_event( project_config, experiment, variation_id, flag_key, rule_key, rule_type, @@ -365,7 +436,7 @@ def _get_feature_variable_for_type( if decision.variation: - feature_enabled = decision.variation.featureEnabled + feature_enabled = self._get_feature_enabled(decision.variation) if feature_enabled: variable_value = project_config.get_variable_value_for_variation(variable, decision.variation) self.logger.info( @@ -383,10 +454,10 @@ def _get_feature_variable_for_type( f'Returning default value for variable "{variable_key}" of feature flag "{feature_key}".' ) - if decision.source == enums.DecisionSources.FEATURE_TEST: + if decision.source in (enums.DecisionSources.FEATURE_TEST, enums.DecisionSources.HOLDOUT): source_info = { 'experiment_key': decision.experiment.key if decision.experiment else None, - 'variation_key': decision.variation.key if decision.variation else None, + 'variation_key': self._get_variation_key(decision.variation), } try: @@ -454,7 +525,7 @@ def _get_all_feature_variables_for_type( if decision.variation: - feature_enabled = decision.variation.featureEnabled + feature_enabled = self._get_feature_enabled(decision.variation) if feature_enabled: self.logger.info( f'Feature "{feature_key}" is enabled for user "{user_id}".' @@ -490,7 +561,7 @@ def _get_all_feature_variables_for_type( if decision.source == enums.DecisionSources.FEATURE_TEST: source_info = { 'experiment_key': decision.experiment.key if decision.experiment else None, - 'variation_key': decision.variation.key if decision.variation else None, + 'variation_key': self._get_variation_key(decision.variation), } self.notification_center.send_notifications( @@ -555,7 +626,8 @@ def activate(self, experiment_key: str, user_id: str, attributes: Optional[UserA self._send_impression_event(project_config, experiment, variation, '', experiment.key, enums.DecisionSources.EXPERIMENT, True, user_id, attributes) - return variation.key + # Handle both Variation entity and VariationDict + return variation['key'] if isinstance(variation, dict) else variation.key def track( self, event_key: str, user_id: str, @@ -663,7 +735,7 @@ def get_variation( variation = variation_result['variation'] user_profile_tracker.save_user_profile() if variation: - variation_key = variation.key + variation_key = self._get_variation_key(variation) if project_config.is_feature_experiment(experiment.id): decision_notification_type = enums.DecisionNotificationTypes.FEATURE_TEST @@ -728,7 +800,7 @@ def is_feature_enabled(self, feature_key: str, user_id: str, attributes: Optiona is_source_rollout = decision.source == enums.DecisionSources.ROLLOUT if decision.variation: - if decision.variation.featureEnabled is True: + if self._get_feature_enabled(decision.variation) is True: feature_enabled = True if (is_source_rollout or not decision.variation) and project_config.get_send_flag_decisions_value(): @@ -741,7 +813,7 @@ def is_feature_enabled(self, feature_key: str, user_id: str, attributes: Optiona if is_source_experiment and decision.variation and decision.experiment: source_info = { 'experiment_key': decision.experiment.key, - 'variation_key': decision.variation.key, + 'variation_key': self._get_variation_key(decision.variation), } self._send_impression_event( project_config, decision.experiment, decision.variation, feature.key, decision.experiment.key, @@ -1050,7 +1122,10 @@ def get_forced_variation(self, experiment_key: str, user_id: str) -> Optional[st return None forced_variation, _ = self.decision_service.get_forced_variation(project_config, experiment_key, user_id) - return forced_variation.key if forced_variation else None + if forced_variation: + # Handle both Variation entity and VariationDict + return forced_variation['key'] if isinstance(forced_variation, dict) else forced_variation.key + return None def get_optimizely_config(self) -> Optional[OptimizelyConfig]: """ Gets OptimizelyConfig instance for the current project config. @@ -1175,7 +1250,7 @@ def _create_optimizely_decision( user_id = user_context.user_id feature_enabled = False if flag_decision.variation is not None: - if flag_decision.variation.featureEnabled: + if self._get_feature_enabled(flag_decision.variation): feature_enabled = True self.logger.info(f'Feature {flag_key} is enabled for user {user_id} {feature_enabled}"') @@ -1190,9 +1265,10 @@ def _create_optimizely_decision( feature_flag = project_config.feature_key_map.get(flag_key) # Send impression event if Decision came from a feature - # test and decide options doesn't include disableDecisionEvent if OptimizelyDecideOption.DISABLE_DECISION_EVENT not in decide_options: - if decision_source == DecisionSources.FEATURE_TEST or project_config.send_flag_decisions: + if (decision_source == DecisionSources.FEATURE_TEST or + decision_source == DecisionSources.HOLDOUT or + project_config.send_flag_decisions): self._send_impression_event(project_config, flag_decision.experiment, flag_decision.variation, @@ -1224,11 +1300,7 @@ def _create_optimizely_decision( all_variables[variable_key] = actual_value should_include_reasons = OptimizelyDecideOption.INCLUDE_REASONS in decide_options - variation_key = ( - flag_decision.variation.key - if flag_decision is not None and flag_decision.variation is not None - else None - ) + variation_key = self._get_variation_key(flag_decision.variation) experiment_id = None variation_id = None @@ -1241,7 +1313,7 @@ def _create_optimizely_decision( try: if flag_decision.variation is not None: - variation_id = flag_decision.variation.id + variation_id = self._get_variation_id(flag_decision.variation) except AttributeError: self.logger.warning("flag_decision.variation has no attribute 'id'") diff --git a/optimizely/optimizely_factory.py b/optimizely/optimizely_factory.py index ae466979..85dccd4f 100644 --- a/optimizely/optimizely_factory.py +++ b/optimizely/optimizely_factory.py @@ -22,6 +22,9 @@ from .event_dispatcher import EventDispatcher, CustomEventDispatcher from .notification_center import NotificationCenter from .optimizely import Optimizely +from .odp.lru_cache import LRUCache +from .cmab.cmab_client import DefaultCmabClient, CmabRetryConfig +from .cmab.cmab_service import DefaultCmabService, CmabCacheValue, DEFAULT_CMAB_CACHE_TIMEOUT, DEFAULT_CMAB_CACHE_SIZE if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime @@ -36,6 +39,9 @@ class OptimizelyFactory: max_event_flush_interval: Optional[int] = None polling_interval: Optional[float] = None blocking_timeout: Optional[int] = None + cmab_cache_size: Optional[int] = None + cmab_cache_ttl: Optional[int] = None + cmab_custom_cache: Optional[LRUCache[str, CmabCacheValue]] = None @staticmethod def set_batch_size(batch_size: int) -> int: @@ -75,6 +81,51 @@ def set_blocking_timeout(blocking_timeout: int) -> int: OptimizelyFactory.blocking_timeout = blocking_timeout return OptimizelyFactory.blocking_timeout + @staticmethod + def set_cmab_cache_size(cache_size: int, logger: Optional[optimizely_logger.Logger] = None) -> Optional[int]: + """ Convenience method for setting the maximum number of items in CMAB cache. + Args: + cache_size: Maximum number of items in CMAB cache. + logger: Optional logger for logging messages. + """ + logger = logger or optimizely_logger.NoOpLogger() + + if not isinstance(cache_size, int) or cache_size <= 0: + logger.error( + f"CMAB cache size is invalid, setting to default size {DEFAULT_CMAB_CACHE_SIZE}." + ) + return None + + OptimizelyFactory.cmab_cache_size = cache_size + return OptimizelyFactory.cmab_cache_size + + @staticmethod + def set_cmab_cache_ttl(cache_ttl: int, logger: Optional[optimizely_logger.Logger] = None) -> Optional[int]: + """ Convenience method for setting CMAB cache TTL. + Args: + cache_ttl: Time in seconds for cache entries to live. + logger: Optional logger for logging messages. + """ + logger = logger or optimizely_logger.NoOpLogger() + + if not isinstance(cache_ttl, (int, float)) or cache_ttl <= 0: + logger.error( + f"CMAB cache TTL is invalid, setting to default TTL {DEFAULT_CMAB_CACHE_TIMEOUT}." + ) + return None + + OptimizelyFactory.cmab_cache_ttl = int(cache_ttl) + return OptimizelyFactory.cmab_cache_ttl + + @staticmethod + def set_cmab_custom_cache(custom_cache: LRUCache[str, CmabCacheValue]) -> LRUCache[str, CmabCacheValue]: + """ Convenience method for setting custom CMAB cache. + Args: + custom_cache: Cache implementation with lookup, save, remove, and reset methods. + """ + OptimizelyFactory.cmab_custom_cache = custom_cache + return OptimizelyFactory.cmab_custom_cache + @staticmethod def default_instance(sdk_key: str, datafile: Optional[str] = None) -> Optimizely: """ Returns a new optimizely instance.. @@ -104,9 +155,17 @@ def default_instance(sdk_key: str, datafile: Optional[str] = None) -> Optimizely notification_center=notification_center, ) + # Initialize CMAB components + cmab_client = DefaultCmabClient(retry_config=CmabRetryConfig(), logger=logger) + cmab_cache = OptimizelyFactory.cmab_custom_cache or LRUCache( + OptimizelyFactory.cmab_cache_size or DEFAULT_CMAB_CACHE_SIZE, + OptimizelyFactory.cmab_cache_ttl or DEFAULT_CMAB_CACHE_TIMEOUT + ) + cmab_service = DefaultCmabService(cmab_cache, cmab_client, logger) + optimizely = Optimizely( datafile, None, logger, error_handler, None, None, sdk_key, config_manager, notification_center, - event_processor + event_processor, cmab_service=cmab_service ) return optimizely @@ -174,7 +233,16 @@ def custom_instance( notification_center=notification_center, ) + # Initialize CMAB components + cmab_client = DefaultCmabClient(retry_config=CmabRetryConfig(), logger=logger) + cmab_cache = OptimizelyFactory.cmab_custom_cache or LRUCache( + OptimizelyFactory.cmab_cache_size or DEFAULT_CMAB_CACHE_SIZE, + OptimizelyFactory.cmab_cache_ttl or DEFAULT_CMAB_CACHE_TIMEOUT + ) + cmab_service = DefaultCmabService(cmab_cache, cmab_client, logger) + return Optimizely( datafile, event_dispatcher, logger, error_handler, skip_json_validation, user_profile_service, - sdk_key, config_manager, notification_center, event_processor, settings=settings + sdk_key, config_manager, notification_center, event_processor, settings=settings, + cmab_service=cmab_service ) diff --git a/optimizely/project_config.py b/optimizely/project_config.py index 446d1e2f..74442d7a 100644 --- a/optimizely/project_config.py +++ b/optimizely/project_config.py @@ -12,7 +12,7 @@ # limitations under the License. from __future__ import annotations import json -from typing import TYPE_CHECKING, Optional, Type, TypeVar, cast, Any, Iterable, List +from typing import TYPE_CHECKING, Optional, Type, TypeVar, Union, cast, Any, Iterable, List from sys import version_info from . import entities @@ -24,11 +24,12 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final # type: ignore + from typing import Final if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime from .logger import Logger + from .helpers.types import VariationDict SUPPORTED_VERSIONS = [ @@ -88,6 +89,48 @@ def __init__(self, datafile: str | bytes, logger: Logger, error_handler: Any): region_value = config.get('region') self.region: str = region_value or 'US' + # Parse holdouts from datafile and convert to Holdout entities + holdouts_data: list[types.HoldoutDict] = config.get('holdouts', []) + self.holdouts: list[entities.Holdout] = [] + self.holdout_id_map: dict[str, entities.Holdout] = {} + self.global_holdouts: list[entities.Holdout] = [] + self.included_holdouts: dict[str, list[entities.Holdout]] = {} + self.excluded_holdouts: dict[str, list[entities.Holdout]] = {} + self.flag_holdouts_map: dict[str, list[entities.Holdout]] = {} + + # Convert holdout dicts to Holdout entities + for holdout_data in holdouts_data: + # Create Holdout entity + holdout = entities.Holdout(**holdout_data) + self.holdouts.append(holdout) + + # Only process Running holdouts but doing it here for efficiency like the original Python implementation) + if not holdout.is_activated: + continue + + # Map by ID for quick lookup + self.holdout_id_map[holdout.id] = holdout + + # Categorize as global vs flag-specific + # Global holdouts: apply to all flags unless explicitly excluded + # Flag-specific holdouts: only apply to explicitly included flags + if not holdout.includedFlags: + # This is a global holdout + self.global_holdouts.append(holdout) + + # Track which flags this global holdout excludes + if holdout.excludedFlags: + for flag_id in holdout.excludedFlags: + if flag_id not in self.excluded_holdouts: + self.excluded_holdouts[flag_id] = [] + self.excluded_holdouts[flag_id].append(holdout) + else: + # This holdout applies to specific flags only + for flag_id in holdout.includedFlags: + if flag_id not in self.included_holdouts: + self.included_holdouts[flag_id] = [] + self.included_holdouts[flag_id].append(holdout) + # Utility maps for quick lookup self.group_id_map: dict[str, entities.Group] = self._generate_key_map(self.groups, 'id', entities.Group) self.experiment_id_map: dict[str, entities.Experiment] = self._generate_key_map( @@ -139,11 +182,11 @@ def __init__(self, datafile: str | bytes, logger: Logger, error_handler: Any): self.all_segments += audience.get_segments() self.experiment_key_map: dict[str, entities.Experiment] = {} - self.variation_key_map: dict[str, dict[str, entities.Variation]] = {} - self.variation_id_map: dict[str, dict[str, entities.Variation]] = {} + self.variation_key_map: dict[str, dict[str, Union[entities.Variation, VariationDict]]] = {} + self.variation_id_map: dict[str, dict[str, Union[entities.Variation, VariationDict]]] = {} self.variation_variable_usage_map: dict[str, dict[str, entities.Variation.VariableUsage]] = {} - self.variation_id_map_by_experiment_id: dict[str, dict[str, entities.Variation]] = {} - self.variation_key_map_by_experiment_id: dict[str, dict[str, entities.Variation]] = {} + self.variation_id_map_by_experiment_id: dict[str, dict[str, Union[entities.Variation, VariationDict]]] = {} + self.variation_key_map_by_experiment_id: dict[str, dict[str, Union[entities.Variation, VariationDict]]] = {} self.flag_variations_map: dict[str, list[entities.Variation]] = {} for experiment in self.experiment_id_map.values(): @@ -157,11 +200,13 @@ def __init__(self, datafile: str | bytes, logger: Logger, error_handler: Any): self.variation_key_map_by_experiment_id[experiment.id] = {} for variation in self.variation_key_map[experiment.key].values(): - self.variation_id_map[experiment.key][variation.id] = variation - self.variation_id_map_by_experiment_id[experiment.id][variation.id] = variation - self.variation_key_map_by_experiment_id[experiment.id][variation.key] = variation - self.variation_variable_usage_map[variation.id] = self._generate_key_map( - variation.variables, 'id', entities.Variation.VariableUsage + # Cast is safe here because experiments always use Variation entities, not VariationDict + var = cast(entities.Variation, variation) + self.variation_id_map[experiment.key][var.id] = var + self.variation_id_map_by_experiment_id[experiment.id][var.id] = var + self.variation_key_map_by_experiment_id[experiment.id][var.key] = var + self.variation_variable_usage_map[var.id] = self._generate_key_map( + var.variables, 'id', entities.Variation.VariableUsage ) self.feature_key_map = self._generate_key_map(self.feature_flags, 'key', entities.FeatureFlag) @@ -186,6 +231,23 @@ def __init__(self, datafile: str | bytes, logger: Logger, error_handler: Any): # Add this experiment in experiment-feature map. self.experiment_feature_map[exp_id] = [feature.id] rules.append(self.experiment_id_map[exp_id]) + + flag_id = feature.id + applicable_holdouts: list[entities.Holdout] = [] + + # Add global holdouts first, excluding any that are explicitly excluded for this flag + excluded_holdouts = self.excluded_holdouts.get(flag_id, []) + for holdout in self.global_holdouts: + if holdout not in excluded_holdouts: + applicable_holdouts.append(holdout) + + # Add flag-specific local holdouts AFTER global holdouts + if flag_id in self.included_holdouts: + applicable_holdouts.extend(self.included_holdouts[flag_id]) + + if applicable_holdouts: + self.flag_holdouts_map[feature.key] = applicable_holdouts + rollout = None if len(feature.rolloutId) == 0 else self.rollout_id_map[feature.rolloutId] if rollout: for exp in rollout.experiments: @@ -195,10 +257,29 @@ def __init__(self, datafile: str | bytes, logger: Logger, error_handler: Any): # variation_id_map_by_experiment_id gives variation entity object while # experiment_id_map will give us dictionary for rule_variation in self.variation_id_map_by_experiment_id[rule.id].values(): - if len(list(filter(lambda variation: variation.id == rule_variation.id, variations))) == 0: - variations.append(rule_variation) + # Cast is safe here because rollout rules use Variation entities + rule_var = cast(entities.Variation, rule_variation) + if len(list(filter(lambda variation: variation.id == rule_var.id, variations))) == 0: + variations.append(rule_var) self.flag_variations_map[feature.key] = variations + # Process holdout variations are converted to Variation entities just like experiment variations + if self.holdouts: + for holdout in self.holdouts: + # Initialize variation maps for this holdout + self.variation_key_map[holdout.key] = {} + self.variation_id_map[holdout.key] = {} + self.variation_id_map_by_experiment_id[holdout.id] = {} + self.variation_key_map_by_experiment_id[holdout.id] = {} + + if holdout.variations: + for variation_dict in holdout.variations: + # Map variations by key and ID using dict format + self.variation_key_map[holdout.key][variation_dict['key']] = variation_dict + self.variation_id_map[holdout.key][variation_dict['id']] = variation_dict + self.variation_key_map_by_experiment_id[holdout.id][variation_dict['key']] = variation_dict + self.variation_id_map_by_experiment_id[holdout.id][variation_dict['id']] = variation_dict + @staticmethod def _generate_key_map( entity_list: Iterable[Any], key: str, entity_class: Type[EntityClass], first_value: bool = False @@ -415,7 +496,9 @@ def get_audience(self, audience_id: str) -> Optional[entities.Audience]: self.error_handler.handle_error(exceptions.InvalidAudienceException((enums.Errors.INVALID_AUDIENCE))) return None - def get_variation_from_key(self, experiment_key: str, variation_key: str) -> Optional[entities.Variation]: + def get_variation_from_key( + self, experiment_key: str, variation_key: str + ) -> Optional[Union[entities.Variation, VariationDict]]: """ Get variation given experiment and variation key. Args: @@ -442,7 +525,9 @@ def get_variation_from_key(self, experiment_key: str, variation_key: str) -> Opt self.error_handler.handle_error(exceptions.InvalidExperimentException(enums.Errors.INVALID_EXPERIMENT_KEY)) return None - def get_variation_from_id(self, experiment_key: str, variation_id: str) -> Optional[entities.Variation]: + def get_variation_from_id( + self, experiment_key: str, variation_id: str + ) -> Optional[Union[entities.Variation, VariationDict]]: """ Get variation given experiment and variation ID. Args: @@ -583,7 +668,7 @@ def get_rollout_from_id(self, rollout_id: str) -> Optional[entities.Layer]: return None def get_variable_value_for_variation( - self, variable: Optional[entities.Variable], variation: Optional[entities.Variation] + self, variable: Optional[entities.Variable], variation: Optional[Union[entities.Variation, VariationDict]] ) -> Optional[str]: """ Get the variable value for the given variation. @@ -597,12 +682,21 @@ def get_variable_value_for_variation( if not variable or not variation: return None - if variation.id not in self.variation_variable_usage_map: - self.logger.error(f'Variation with ID "{variation.id}" is not in the datafile.') + + # Extract variation ID from either Variation entity or dict + if isinstance(variation, dict): + variation_id = variation.get('id') + if not variation_id: + return None + else: + variation_id = variation.id + + if variation_id not in self.variation_variable_usage_map: + self.logger.error(f'Variation with ID "{variation_id}" is not in the datafile.') return None # Get all variable usages for the given variation - variable_usages = self.variation_variable_usage_map[variation.id] + variable_usages = self.variation_variable_usage_map[variation_id] # Find usage in given variation variable_usage = None @@ -611,7 +705,6 @@ def get_variable_value_for_variation( if variable_usage: variable_value = variable_usage.value - else: variable_value = variable.defaultValue @@ -680,7 +773,7 @@ def is_feature_experiment(self, experiment_id: str) -> bool: def get_variation_from_id_by_experiment_id( self, experiment_id: str, variation_id: str - ) -> Optional[entities.Variation]: + ) -> Optional[Union[entities.Variation, VariationDict]]: """ Gets variation from variation id and specific experiment id Returns: @@ -699,7 +792,7 @@ def get_variation_from_id_by_experiment_id( def get_variation_from_key_by_experiment_id( self, experiment_id: str, variation_key: str - ) -> Optional[entities.Variation]: + ) -> Optional[Union[entities.Variation, VariationDict]]: """ Gets variation from variation key and specific experiment id Returns: @@ -752,3 +845,34 @@ def get_flag_variation( return variation return None + + def get_holdouts_for_flag(self, flag_key: str) -> list[entities.Holdout]: + """ Helper method to get holdouts from an applied feature flag. + + Args: + flag_key: Key of the feature flag. + + Returns: + The holdouts that apply for a specific flag as Holdout entity objects. + """ + if not self.holdouts: + return [] + + return self.flag_holdouts_map.get(flag_key, []) + + def get_holdout(self, holdout_id: str) -> Optional[entities.Holdout]: + """ Helper method to get holdout from holdout ID. + + Args: + holdout_id: ID of the holdout. + + Returns: + The holdout corresponding to the provided holdout ID as a Holdout entity object. + """ + holdout = self.holdout_id_map.get(holdout_id) + + if holdout: + return holdout + + self.logger.error(f'Holdout with ID "{holdout_id}" not found.') + return None diff --git a/optimizely/user_profile.py b/optimizely/user_profile.py index f5ded013..e3a56195 100644 --- a/optimizely/user_profile.py +++ b/optimizely/user_profile.py @@ -19,7 +19,7 @@ if version_info < (3, 8): from typing_extensions import Final else: - from typing import Final, TYPE_CHECKING # type: ignore + from typing import Final, TYPE_CHECKING if TYPE_CHECKING: # prevent circular dependenacy by skipping import at runtime diff --git a/optimizely/version.py b/optimizely/version.py index 4f0f20c6..aef5b367 100644 --- a/optimizely/version.py +++ b/optimizely/version.py @@ -11,5 +11,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version_info = (5, 2, 0) +version_info = (5, 4, 0) __version__ = '.'.join(str(v) for v in version_info) diff --git a/requirements/core.txt b/requirements/core.txt index 7cbfe29f..ea81c17b 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -2,3 +2,4 @@ jsonschema>=3.2.0 pyrsistent>=0.16.0 requests>=2.21 idna>=2.10 +rpds-py<0.20.0; python_version < '3.11' diff --git a/requirements/typing.txt b/requirements/typing.txt index ba65f536..4c01897b 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -1,4 +1,5 @@ mypy types-jsonschema types-requests -types-Flask \ No newline at end of file +types-Flask +rpds-py<0.20.0; python_version < '3.11' \ No newline at end of file diff --git a/tests/test_bucketing_holdout.py b/tests/test_bucketing_holdout.py new file mode 100644 index 00000000..e3d9bcb3 --- /dev/null +++ b/tests/test_bucketing_holdout.py @@ -0,0 +1,360 @@ +# Copyright 2025 Optimizely and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +from unittest import mock + +from optimizely import bucketer +from optimizely import error_handler +from optimizely import logger +from optimizely import optimizely as optimizely_module +from tests import base + + +class TestBucketer(bucketer.Bucketer): + """Helper class for testing with controlled bucket values.""" + + def __init__(self): + super().__init__() + self._bucket_values = [] + self._bucket_index = 0 + + def set_bucket_values(self, values): + """Set predetermined bucket values for testing.""" + self._bucket_values = values + self._bucket_index = 0 + + def _generate_bucket_value(self, bucketing_id): + """Override to return controlled bucket values.""" + if not self._bucket_values: + return super()._generate_bucket_value(bucketing_id) + + value = self._bucket_values[self._bucket_index] + self._bucket_index = (self._bucket_index + 1) % len(self._bucket_values) + return value + + +class BucketerHoldoutTest(base.BaseTest): + """Tests for Optimizely Bucketer with Holdouts.""" + + def setUp(self): + base.BaseTest.setUp(self) + self.error_handler = error_handler.NoOpErrorHandler() + self.mock_logger = mock.MagicMock(spec=logger.Logger) + + # Create a config dict with holdouts that have variations and traffic allocation + config_dict_with_holdouts = self.config_dict.copy() + + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'var_1', + 'key': 'control', + 'variables': [] + }, + { + 'id': 'var_2', + 'key': 'treatment', + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'var_1', + 'endOfRange': 5000 + }, + { + 'entityId': 'var_2', + 'endOfRange': 10000 + } + ] + }, + { + 'id': 'holdout_empty_1', + 'key': 'empty_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [], + 'trafficAllocation': [] + } + ] + + # Convert to JSON and create config + config_json = json.dumps(config_dict_with_holdouts) + opt_obj = optimizely_module.Optimizely(config_json) + self.config = opt_obj.config_manager.get_config() + + self.test_bucketer = TestBucketer() + self.test_user_id = 'test_user_id' + self.test_bucketing_id = 'test_bucketing_id' + + # Verify that the config contains holdouts + self.assertIsNotNone(self.config.holdouts) + self.assertGreater(len(self.config.holdouts), 0) + + def test_bucket_user_within_valid_traffic_allocation_range(self): + """Should bucket user within valid traffic allocation range.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Set bucket value to be within first variation's traffic allocation (0-5000 range) + self.test_bucketer.set_bucket_values([2500]) + + variation, reasons = self.test_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNotNone(variation) + self.assertEqual(variation['id'], 'var_1') + self.assertEqual(variation['key'], 'control') + + def test_bucket_returns_none_when_user_outside_traffic_allocation(self): + """Should return None when user is outside traffic allocation range.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Modify traffic allocation to be smaller + modified_holdout = copy.deepcopy(holdout) + modified_holdout['trafficAllocation'] = [ + { + 'entityId': 'var_1', + 'endOfRange': 1000 + } + ] + + # Set bucket value outside traffic allocation range + self.test_bucketer.set_bucket_values([1500]) + + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_returns_none_when_holdout_has_no_traffic_allocation(self): + """Should return None when holdout has no traffic allocation.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Clear traffic allocation + modified_holdout = copy.deepcopy(holdout) + modified_holdout['trafficAllocation'] = [] + + self.test_bucketer.set_bucket_values([5000]) + + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_returns_none_with_invalid_variation_id(self): + """Should return None when traffic allocation points to invalid variation ID.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Set traffic allocation to point to non-existent variation + modified_holdout = copy.deepcopy(holdout) + modified_holdout['trafficAllocation'] = [ + { + 'entityId': 'invalid_variation_id', + 'endOfRange': 10000 + } + ] + + self.test_bucketer.set_bucket_values([5000]) + + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_returns_none_when_holdout_has_no_variations(self): + """Should return None when holdout has no variations.""" + holdout = self.config.get_holdout('holdout_empty_1') + self.assertIsNotNone(holdout) + self.assertEqual(len(holdout.get('variations', [])), 0) + + self.test_bucketer.set_bucket_values([5000]) + + variation, reasons = self.test_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_returns_none_with_empty_key(self): + """Should return None when holdout has empty key.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Clear holdout key + modified_holdout = copy.deepcopy(holdout) + modified_holdout['key'] = '' + + self.test_bucketer.set_bucket_values([5000]) + + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_returns_none_with_null_key(self): + """Should return None when holdout has null key.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Set holdout key to None + modified_holdout = copy.deepcopy(holdout) + modified_holdout['key'] = None + + self.test_bucketer.set_bucket_values([5000]) + + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_user_into_first_variation_with_multiple_variations(self): + """Should bucket user into first variation with multiple variations.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Verify holdout has multiple variations + self.assertGreaterEqual(len(holdout['variations']), 2) + + # Test user buckets into first variation + self.test_bucketer.set_bucket_values([2500]) + variation, reasons = self.test_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNotNone(variation) + self.assertEqual(variation['id'], 'var_1') + self.assertEqual(variation['key'], 'control') + + def test_bucket_user_into_second_variation_with_multiple_variations(self): + """Should bucket user into second variation with multiple variations.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Verify holdout has multiple variations + self.assertGreaterEqual(len(holdout['variations']), 2) + self.assertEqual(holdout['variations'][0]['id'], 'var_1') + self.assertEqual(holdout['variations'][1]['id'], 'var_2') + + # Test user buckets into second variation (bucket value 7500 should be in 5000-10000 range) + self.test_bucketer.set_bucket_values([7500]) + variation, reasons = self.test_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNotNone(variation) + self.assertEqual(variation['id'], 'var_2') + self.assertEqual(variation['key'], 'treatment') + + def test_bucket_handles_edge_case_boundary_values(self): + """Should handle edge case boundary values correctly.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Modify traffic allocation to be 5000 (50%) + modified_holdout = copy.deepcopy(holdout) + modified_holdout['trafficAllocation'] = [ + { + 'entityId': 'var_1', + 'endOfRange': 5000 + } + ] + + # Test exact boundary value (should be included) + self.test_bucketer.set_bucket_values([4999]) + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNotNone(variation) + self.assertEqual(variation['id'], 'var_1') + + # Test value just outside boundary (should not be included) + self.test_bucketer.set_bucket_values([5000]) + variation, reasons = self.test_bucketer.bucket( + self.config, modified_holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNone(variation) + + def test_bucket_produces_consistent_results_with_same_inputs(self): + """Should produce consistent results with same inputs.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Create a real bucketer (not test bucketer) for consistent hashing + real_bucketer = bucketer.Bucketer() + variation1, reasons1 = real_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + variation2, reasons2 = real_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + # Results should be identical + if variation1: + self.assertIsNotNone(variation2) + self.assertEqual(variation1['id'], variation2['id']) + self.assertEqual(variation1['key'], variation2['key']) + else: + self.assertIsNone(variation2) + + def test_bucket_handles_different_bucketing_ids_without_exceptions(self): + """Should handle different bucketing IDs without exceptions.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + # Create a real bucketer (not test bucketer) for real hashing behavior + real_bucketer = bucketer.Bucketer() + + # These calls should not raise exceptions + try: + real_bucketer.bucket(self.config, holdout, self.test_user_id, 'bucketingId1') + real_bucketer.bucket(self.config, holdout, self.test_user_id, 'bucketingId2') + except Exception as e: + self.fail(f'Bucketing raised an exception: {e}') + + def test_bucket_populates_decision_reasons_properly(self): + """Should populate decision reasons properly.""" + holdout = self.config.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + + self.test_bucketer.set_bucket_values([5000]) + variation, reasons = self.test_bucketer.bucket( + self.config, holdout, self.test_user_id, self.test_bucketing_id + ) + + self.assertIsNotNone(reasons) + self.assertIsInstance(reasons, list) + # Decision reasons should be populated from the bucketing process diff --git a/tests/test_cmab_client.py b/tests/test_cmab_client.py index 3aac5fd9..3613da76 100644 --- a/tests/test_cmab_client.py +++ b/tests/test_cmab_client.py @@ -245,3 +245,81 @@ def test_fetch_decision_exhausts_all_retry_attempts(self, mock_sleep): self.mock_logger.error.assert_called_with( Errors.CMAB_FETCH_FAILED.format('Exhausted all retries for CMAB request.') ) + + def test_custom_prediction_endpoint(self): + """Test that custom prediction endpoint is used correctly.""" + custom_endpoint = "https://custom.endpoint.com/predict/{}" + client = DefaultCmabClient( + http_client=self.mock_http_client, + logger=self.mock_logger, + prediction_endpoint=custom_endpoint + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'predictions': [{'variation_id': 'abc123'}] + } + self.mock_http_client.post.return_value = mock_response + + result = client.fetch_decision(self.rule_id, self.user_id, self.attributes, self.cmab_uuid) + + self.assertEqual(result, 'abc123') + expected_custom_url = custom_endpoint.format(self.rule_id) + self.mock_http_client.post.assert_called_once_with( + expected_custom_url, + data=json.dumps(self.expected_body), + headers=self.expected_headers, + timeout=10.0 + ) + + def test_default_prediction_endpoint(self): + """Test that default prediction endpoint is used when none is provided.""" + client = DefaultCmabClient( + http_client=self.mock_http_client, + logger=self.mock_logger + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'predictions': [{'variation_id': 'def456'}] + } + self.mock_http_client.post.return_value = mock_response + + result = client.fetch_decision(self.rule_id, self.user_id, self.attributes, self.cmab_uuid) + + self.assertEqual(result, 'def456') + # Should use the default production endpoint + self.mock_http_client.post.assert_called_once_with( + self.expected_url, + data=json.dumps(self.expected_body), + headers=self.expected_headers, + timeout=10.0 + ) + + def test_empty_prediction_endpoint_uses_default(self): + """Test that empty string prediction endpoint falls back to default.""" + client = DefaultCmabClient( + http_client=self.mock_http_client, + logger=self.mock_logger, + prediction_endpoint="" + ) + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'predictions': [{'variation_id': 'ghi789'}] + } + self.mock_http_client.post.return_value = mock_response + + result = client.fetch_decision(self.rule_id, self.user_id, self.attributes, self.cmab_uuid) + + self.assertEqual(result, 'ghi789') + # Should use the default production endpoint when empty string is provided + self.mock_http_client.post.assert_called_once_with( + self.expected_url, + data=json.dumps(self.expected_body), + headers=self.expected_headers, + timeout=10.0 + ) diff --git a/tests/test_cmab_service.py b/tests/test_cmab_service.py index 0b3c593a..b26e277f 100644 --- a/tests/test_cmab_service.py +++ b/tests/test_cmab_service.py @@ -12,7 +12,7 @@ # limitations under the License. import unittest from unittest.mock import MagicMock -from optimizely.cmab.cmab_service import DefaultCmabService +from optimizely.cmab.cmab_service import DefaultCmabService, NUM_LOCK_STRIPES from optimizely.optimizely_user_context import OptimizelyUserContext from optimizely.decision.optimizely_decide_option import OptimizelyDecideOption from optimizely.odp.lru_cache import LRUCache @@ -60,7 +60,7 @@ def test_returns_decision_from_cache_when_valid(self): "cmab_uuid": "uuid-123" } - decision = self.cmab_service.get_decision( + decision, _ = self.cmab_service.get_decision( self.mock_project_config, self.mock_user_context, "exp1", [] ) @@ -72,7 +72,7 @@ def test_ignores_cache_when_option_given(self): self.mock_cmab_client.fetch_decision.return_value = "varB" expected_attributes = {"age": 25, "location": "USA"} - decision = self.cmab_service.get_decision( + decision, _ = self.cmab_service.get_decision( self.mock_project_config, self.mock_user_context, "exp1", @@ -105,7 +105,7 @@ def test_invalidates_user_cache_when_option_given(self): def test_resets_cache_when_option_given(self): self.mock_cmab_client.fetch_decision.return_value = "varD" - decision = self.cmab_service.get_decision( + decision, _ = self.cmab_service.get_decision( self.mock_project_config, self.mock_user_context, "exp1", @@ -128,7 +128,7 @@ def test_new_decision_when_hash_changes(self): expected_hash = self.cmab_service._hash_attributes(expected_attribute) expected_key = self.cmab_service._get_cache_key("user123", "exp1") - decision = self.cmab_service.get_decision(self.mock_project_config, self.mock_user_context, "exp1", []) + decision, _ = self.cmab_service.get_decision(self.mock_project_config, self.mock_user_context, "exp1", []) self.mock_cmab_cache.remove.assert_called_once_with(expected_key) self.mock_cmab_cache.save.assert_called_once_with( expected_key, @@ -171,7 +171,7 @@ def test_only_cmab_attributes_passed_to_client(self): } self.mock_cmab_client.fetch_decision.return_value = "varF" - decision = self.cmab_service.get_decision( + decision, _ = self.cmab_service.get_decision( self.mock_project_config, self.mock_user_context, "exp1", @@ -185,3 +185,41 @@ def test_only_cmab_attributes_passed_to_client(self): {"age": 25, "location": "USA"}, decision["cmab_uuid"] ) + + def test_same_user_rule_combination_uses_consistent_lock(self): + """Verifies that the same user/rule combination always uses the same lock index""" + user_id = "test_user" + rule_id = "test_rule" + + # Get lock index multiple times + index1 = self.cmab_service._get_lock_index(user_id, rule_id) + index2 = self.cmab_service._get_lock_index(user_id, rule_id) + index3 = self.cmab_service._get_lock_index(user_id, rule_id) + + # All should be the same + self.assertEqual(index1, index2, "Same user/rule should always use same lock") + self.assertEqual(index2, index3, "Same user/rule should always use same lock") + + def test_lock_striping_distribution(self): + """Verifies that different user/rule combinations use different locks to allow for better concurrency""" + test_cases = [ + ("user1", "rule1"), + ("user2", "rule1"), + ("user1", "rule2"), + ("user3", "rule3"), + ("user4", "rule4"), + ] + + lock_indices = set() + for user_id, rule_id in test_cases: + index = self.cmab_service._get_lock_index(user_id, rule_id) + + # Verify index is within expected range + self.assertGreaterEqual(index, 0, "Lock index should be non-negative") + self.assertLess(index, NUM_LOCK_STRIPES, "Lock index should be less than NUM_LOCK_STRIPES") + + lock_indices.add(index) + + # We should have multiple different lock indices (though not necessarily all unique due to hash collisions) + self.assertGreater(len(lock_indices), 1, + "Different user/rule combinations should generally use different locks") diff --git a/tests/test_config.py b/tests/test_config.py index a6e828c2..81228feb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1375,3 +1375,173 @@ def test_get_variation_from_key_by_experiment_id_missing(self): variation = project_config.get_variation_from_key_by_experiment_id(experiment_id, variation_key) self.assertIsNone(variation) + + +class HoldoutConfigTest(base.BaseTest): + def setUp(self): + base.BaseTest.setUp(self) + + # Create config with holdouts + config_body_with_holdouts = self.config_dict_with_features.copy() + + # Use correct feature flag IDs from the datafile + boolean_feature_id = '91111' # boolean_single_variable_feature + multi_variate_feature_id = '91114' # test_feature_in_experiment_and_rollout + + config_body_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'global_holdout', + 'status': 'Running', + 'variations': [], + 'trafficAllocation': [], + 'audienceIds': [], + 'includedFlags': [], + 'excludedFlags': [boolean_feature_id] + }, + { + 'id': 'holdout_2', + 'key': 'specific_holdout', + 'status': 'Running', + 'variations': [], + 'trafficAllocation': [], + 'audienceIds': [], + 'includedFlags': [multi_variate_feature_id], + 'excludedFlags': [] + }, + { + 'id': 'holdout_3', + 'key': 'inactive_holdout', + 'status': 'Inactive', + 'variations': [], + 'trafficAllocation': [], + 'audienceIds': [], + 'includedFlags': [boolean_feature_id], + 'excludedFlags': [] + } + ] + + self.config_json_with_holdouts = json.dumps(config_body_with_holdouts) + opt_obj = optimizely.Optimizely(self.config_json_with_holdouts) + self.config_with_holdouts = opt_obj.config_manager.get_config() + + def test_get_holdouts_for_flag__non_existent_flag(self): + """ Test that get_holdouts_for_flag returns empty array for non-existent flag. """ + + holdouts = self.config_with_holdouts.get_holdouts_for_flag('non_existent_flag') + self.assertEqual([], holdouts) + + def test_get_holdouts_for_flag__returns_global_and_specific_holdouts(self): + """ Test that get_holdouts_for_flag returns global holdouts that do not exclude the flag + and specific holdouts that include the flag. """ + + holdouts = self.config_with_holdouts.get_holdouts_for_flag('test_feature_in_experiment_and_rollout') + self.assertEqual(2, len(holdouts)) + + global_holdout = next((h for h in holdouts if h['key'] == 'global_holdout'), None) + self.assertIsNotNone(global_holdout) + self.assertEqual('holdout_1', global_holdout['id']) + + specific_holdout = next((h for h in holdouts if h['key'] == 'specific_holdout'), None) + self.assertIsNotNone(specific_holdout) + self.assertEqual('holdout_2', specific_holdout['id']) + + def test_get_holdouts_for_flag__excludes_global_holdouts_for_excluded_flags(self): + """ Test that get_holdouts_for_flag does not return global holdouts that exclude the flag. """ + + holdouts = self.config_with_holdouts.get_holdouts_for_flag('boolean_single_variable_feature') + self.assertEqual(0, len(holdouts)) + + global_holdout = next((h for h in holdouts if h['key'] == 'global_holdout'), None) + self.assertIsNone(global_holdout) + + def test_get_holdouts_for_flag__caches_results(self): + """ Test that get_holdouts_for_flag caches results for subsequent calls. """ + + holdouts1 = self.config_with_holdouts.get_holdouts_for_flag('test_feature_in_experiment_and_rollout') + holdouts2 = self.config_with_holdouts.get_holdouts_for_flag('test_feature_in_experiment_and_rollout') + + # Should be the same object (cached) + self.assertIs(holdouts1, holdouts2) + self.assertEqual(2, len(holdouts1)) + + def test_get_holdouts_for_flag__returns_only_global_for_non_targeted_flags(self): + """ Test that get_holdouts_for_flag returns only global holdouts for flags not specifically targeted. """ + + holdouts = self.config_with_holdouts.get_holdouts_for_flag('test_feature_in_rollout') + + # Should only include global holdout (not excluded and no specific targeting) + self.assertEqual(1, len(holdouts)) + self.assertEqual('global_holdout', holdouts[0]['key']) + + def test_get_holdout__returns_holdout_for_valid_id(self): + """ Test that get_holdout returns holdout when valid ID is provided. """ + + holdout = self.config_with_holdouts.get_holdout('holdout_1') + self.assertIsNotNone(holdout) + self.assertEqual('holdout_1', holdout['id']) + self.assertEqual('global_holdout', holdout['key']) + self.assertEqual('Running', holdout['status']) + + def test_get_holdout__returns_holdout_regardless_of_status(self): + """ Test that get_holdout returns holdout regardless of status when valid ID is provided. """ + + holdout = self.config_with_holdouts.get_holdout('holdout_2') + self.assertIsNotNone(holdout) + self.assertEqual('holdout_2', holdout['id']) + self.assertEqual('specific_holdout', holdout['key']) + self.assertEqual('Running', holdout['status']) + + def test_get_holdout__returns_none_for_non_existent_id(self): + """ Test that get_holdout returns None for non-existent holdout ID. """ + + holdout = self.config_with_holdouts.get_holdout('non_existent_holdout') + self.assertIsNone(holdout) + + def test_get_holdout__logs_error_when_not_found(self): + """ Test that get_holdout logs error when holdout is not found. """ + + with mock.patch.object(self.config_with_holdouts, 'logger') as mock_logger: + result = self.config_with_holdouts.get_holdout('invalid_holdout_id') + + self.assertIsNone(result) + mock_logger.error.assert_called_once_with('Holdout with ID "invalid_holdout_id" not found.') + + def test_get_holdout__does_not_log_when_found(self): + """ Test that get_holdout does not log when holdout is found. """ + + with mock.patch.object(self.config_with_holdouts, 'logger') as mock_logger: + result = self.config_with_holdouts.get_holdout('holdout_1') + + self.assertIsNotNone(result) + mock_logger.error.assert_not_called() + + def test_holdout_initialization__categorizes_holdouts_properly(self): + """ Test that holdouts are properly categorized during initialization. """ + + self.assertIn('holdout_1', self.config_with_holdouts.holdout_id_map) + self.assertIn('holdout_2', self.config_with_holdouts.holdout_id_map) + # Check if a holdout with ID 'holdout_1' exists in global_holdouts + holdout_ids_in_global = [h.id for h in self.config_with_holdouts.global_holdouts] + self.assertIn('holdout_1', holdout_ids_in_global) + + # Use correct feature flag IDs + boolean_feature_id = '91111' + multi_variate_feature_id = '91114' + + self.assertIn(multi_variate_feature_id, self.config_with_holdouts.included_holdouts) + self.assertTrue(len(self.config_with_holdouts.included_holdouts[multi_variate_feature_id]) > 0) + self.assertNotIn(boolean_feature_id, self.config_with_holdouts.included_holdouts) + + self.assertIn(boolean_feature_id, self.config_with_holdouts.excluded_holdouts) + self.assertTrue(len(self.config_with_holdouts.excluded_holdouts[boolean_feature_id]) > 0) + + def test_holdout_initialization__only_processes_running_holdouts(self): + """ Test that only running holdouts are processed during initialization. """ + + self.assertNotIn('holdout_3', self.config_with_holdouts.holdout_id_map) + self.assertNotIn('holdout_3', self.config_with_holdouts.global_holdouts) + + boolean_feature_id = '91111' + included_for_boolean = self.config_with_holdouts.included_holdouts.get(boolean_feature_id) + self.assertIsNone(included_for_boolean) diff --git a/tests/test_config_manager.py b/tests/test_config_manager.py index 56674381..1930520e 100644 --- a/tests/test_config_manager.py +++ b/tests/test_config_manager.py @@ -517,8 +517,10 @@ def test_fetch_datafile__exception_polling_thread_failed(self, _): log_messages = [args[0] for args, _ in mock_logger.error.call_args_list] for message in log_messages: print(message) - if "Thread for background datafile polling failed. " \ - "Error: timestamp too large to convert to C PyTime_t" not in message: + # Check for key parts of the error message (version-agnostic for Python 3.11+) + if not ("Thread for background datafile polling failed" in message and + "timestamp too large to convert to C" in message and + "PyTime_t" in message): assert False def test_is_running(self, _): diff --git a/tests/test_decision_service.py b/tests/test_decision_service.py index d906a3cf..b38a03b2 100644 --- a/tests/test_decision_service.py +++ b/tests/test_decision_service.py @@ -792,10 +792,13 @@ def test_get_variation_cmab_experiment_user_in_traffic_allocation(self): 'logger') as mock_logger: # Configure CMAB service to return a decision - mock_cmab_service.get_decision.return_value = { - 'variation_id': '111151', - 'cmab_uuid': 'test-cmab-uuid-123' - } + mock_cmab_service.get_decision.return_value = ( + { + 'variation_id': '111151', + 'cmab_uuid': 'test-cmab-uuid-123' + }, + [] # reasons list + ) # Call get_variation with the CMAB experiment variation_result = self.decision_service.get_variation( @@ -1071,6 +1074,124 @@ def test_get_variation_cmab_experiment_with_whitelisted_variation(self): mock_bucket.assert_not_called() mock_cmab_decision.assert_not_called() + def test_get_variation_cmab_experiment_does_not_save_user_profile(self): + """Test that CMAB experiments do not save bucketing decisions to user profile.""" + + # Create a user context + user = optimizely_user_context.OptimizelyUserContext( + optimizely_client=None, + logger=None, + user_id="test_user", + user_attributes={} + ) + + # Create a user profile service and tracker + user_profile_service = user_profile.UserProfileService() + user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service) + + # Create a CMAB experiment + cmab_experiment = entities.Experiment( + '111150', + 'cmab_experiment', + 'Running', + '111150', + [], # No audience IDs + {}, + [ + entities.Variation('111151', 'variation_1'), + entities.Variation('111152', 'variation_2') + ], + [ + {'entityId': '111151', 'endOfRange': 5000}, + {'entityId': '111152', 'endOfRange': 10000} + ], + cmab={'trafficAllocation': 5000} + ) + + with mock.patch('optimizely.helpers.experiment.is_experiment_running', return_value=True), \ + mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', return_value=[True, []]), \ + mock.patch.object(self.decision_service.bucketer, 'bucket_to_entity_id', + return_value=['$', []]), \ + mock.patch.object(self.decision_service, 'cmab_service') as mock_cmab_service, \ + mock.patch.object(self.project_config, 'get_variation_from_id', + return_value=entities.Variation('111151', 'variation_1')), \ + mock.patch.object(user_profile_tracker, 'update_user_profile') as mock_update_profile, \ + mock.patch.object(self.decision_service, 'logger') as mock_logger: + + # Configure CMAB service to return a decision + mock_cmab_service.get_decision.return_value = ( + { + 'variation_id': '111151', + 'cmab_uuid': 'test-cmab-uuid-123' + }, + [] # reasons list + ) + + # Call get_variation with the CMAB experiment and user profile tracker + variation_result = self.decision_service.get_variation( + self.project_config, + cmab_experiment, + user, + user_profile_tracker + ) + variation = variation_result['variation'] + cmab_uuid = variation_result['cmab_uuid'] + + # Verify the variation and cmab_uuid are returned + self.assertEqual(entities.Variation('111151', 'variation_1'), variation) + self.assertEqual('test-cmab-uuid-123', cmab_uuid) + + # Verify user profile was NOT updated for CMAB experiment + mock_update_profile.assert_not_called() + + # Verify debug log was called to explain CMAB exclusion + mock_logger.debug.assert_any_call( + 'Skipping user profile service for CMAB experiment "cmab_experiment". ' + 'CMAB decisions are dynamic and not stored for sticky bucketing.' + ) + + def test_get_variation_standard_experiment_saves_user_profile(self): + """Test that standard (non-CMAB) experiments DO save bucketing decisions to user profile.""" + + user = optimizely_user_context.OptimizelyUserContext( + optimizely_client=None, + logger=None, + user_id="test_user", + user_attributes={} + ) + + # Create a user profile service and tracker + user_profile_service = user_profile.UserProfileService() + user_profile_tracker = user_profile.UserProfileTracker(user.user_id, user_profile_service) + + # Get a standard (non-CMAB) experiment + experiment = self.project_config.get_experiment_from_key("test_experiment") + + with mock.patch('optimizely.decision_service.DecisionService.get_whitelisted_variation', + return_value=[None, []]), \ + mock.patch('optimizely.decision_service.DecisionService.get_stored_variation', + return_value=None), \ + mock.patch('optimizely.helpers.audience.does_user_meet_audience_conditions', + return_value=[True, []]), \ + mock.patch('optimizely.bucketer.Bucketer.bucket', + return_value=[entities.Variation("111129", "variation"), []]), \ + mock.patch.object(user_profile_tracker, 'update_user_profile') as mock_update_profile: + + # Call get_variation with standard experiment and user profile tracker + variation_result = self.decision_service.get_variation( + self.project_config, + experiment, + user, + user_profile_tracker + ) + variation = variation_result['variation'] + + # Verify variation was returned + self.assertEqual(entities.Variation("111129", "variation"), variation) + + # Verify user profile WAS updated for standard experiment + mock_update_profile.assert_called_once_with(experiment, variation) + class FeatureFlagDecisionTests(base.BaseTest): def setUp(self): diff --git a/tests/test_decision_service_holdout.py b/tests/test_decision_service_holdout.py new file mode 100644 index 00000000..e286ba5c --- /dev/null +++ b/tests/test_decision_service_holdout.py @@ -0,0 +1,1470 @@ +# Copyright 2025 Optimizely and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from unittest import mock + +from optimizely import decision_service +from optimizely import error_handler +from optimizely import logger +from optimizely import optimizely as optimizely_module +from optimizely.decision.optimizely_decide_option import OptimizelyDecideOption +from optimizely.helpers import enums +from tests import base + + +class DecisionServiceHoldoutTest(base.BaseTest): + """Tests for Decision Service with Holdouts.""" + + def setUp(self): + base.BaseTest.setUp(self) + self.error_handler = error_handler.NoOpErrorHandler() + self.spy_logger = mock.MagicMock(spec=logger.SimpleLogger) + self.spy_logger.logger = self.spy_logger + self.spy_user_profile_service = mock.MagicMock() + self.spy_cmab_service = mock.MagicMock() + + # Create a config dict with holdouts and feature flags + config_dict_with_holdouts = self.config_dict_with_features.copy() + + # Get feature flag ID for test_feature_in_experiment + test_feature_id = '91111' + + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'holdout_var_1', + 'key': 'holdout_control', + 'variables': [] + }, + { + 'id': 'holdout_var_2', + 'key': 'holdout_treatment', + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'holdout_var_1', + 'endOfRange': 5000 + }, + { + 'entityId': 'holdout_var_2', + 'endOfRange': 10000 + } + ] + }, + { + 'id': 'holdout_2', + 'key': 'excluded_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [test_feature_id], + 'audienceIds': [], + 'variations': [ + { + 'id': 'holdout_var_3', + 'key': 'control', + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'holdout_var_3', + 'endOfRange': 10000 + } + ] + } + ] + + # Convert to JSON and create config + config_json = json.dumps(config_dict_with_holdouts) + self.opt_obj = optimizely_module.Optimizely(config_json) + self.config_with_holdouts = self.opt_obj.config_manager.get_config() + + self.decision_service_with_holdouts = decision_service.DecisionService( + self.spy_logger, + self.spy_user_profile_service, + self.spy_cmab_service + ) + + def tearDown(self): + if hasattr(self, 'opt_obj'): + self.opt_obj.close() + + # get_variations_for_feature_list with holdouts tests + + def test_holdout_active_and_user_bucketed_returns_holdout_decision(self): + """When holdout is active and user is bucketed, should return holdout decision with variation.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + holdout = self.config_with_holdouts.holdouts[0] if self.config_with_holdouts.holdouts else None + self.assertIsNotNone(holdout) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + + # Verify result structure is valid + for decision_result in result: + self.assertIn('decision', decision_result) + self.assertIn('reasons', decision_result) + + def test_holdout_inactive_does_not_bucket_users(self): + """When holdout is inactive, should not bucket users and log appropriate message.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + holdout = self.config_with_holdouts.holdouts[0] if self.config_with_holdouts.holdouts else None + self.assertIsNotNone(holdout) + + # Mock holdout as inactive + original_status = holdout.status + holdout.status = 'Paused' + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_decision_for_flag( + feature_flag, + user_context, + self.config_with_holdouts + ) + + # Assert that result is not nil and has expected structure + self.assertIsNotNone(result) + self.assertIn('decision', result) + self.assertIn('reasons', result) + + # Restore original status + holdout.status = original_status + + def test_user_not_bucketed_into_holdout_executes_successfully(self): + """When user is not bucketed into holdout, should execute successfully with valid result structure.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + # With real bucketer, we can't guarantee specific bucketing results + # but we can verify the method executes successfully + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + def test_holdout_with_user_attributes_for_audience_targeting(self): + """Should evaluate holdout with user attributes.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_attributes = { + 'browser': 'chrome', + 'location': 'us' + } + + user_context = self.opt_obj.create_user_context('testUserId', user_attributes) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + def test_multiple_holdouts_for_single_feature_flag(self): + """Should handle multiple holdouts for a single feature flag.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + def test_holdout_bucketing_with_empty_user_id(self): + """Should allow holdout bucketing with empty user ID.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + # Empty user ID should still be valid for bucketing + user_context = self.opt_obj.create_user_context('', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + + def test_holdout_populates_decision_reasons(self): + """Should populate decision reasons for holdouts.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + + # Find any decision with reasons + decision_with_reasons = next( + (dr for dr in result if dr.get('reasons') and len(dr['reasons']) > 0), + None + ) + + if decision_with_reasons: + self.assertGreater(len(decision_with_reasons['reasons']), 0) + + # get_variation_for_feature with holdouts tests + + def test_user_bucketed_into_holdout_returns_before_experiments(self): + """ + When user is bucketed into holdout, should return holdout decision + before checking experiments or rollouts. + """ + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + # Mock get_holdouts_for_flag to return holdouts + holdout = self.config_with_holdouts.holdouts[0] if self.config_with_holdouts.holdouts else None + self.assertIsNotNone(holdout) + + holdout_variation = holdout.variations[0] + + # Create a holdout decision + mock_holdout_decision = decision_service.Decision( + experiment=holdout, + variation=holdout_variation, + source=enums.DecisionSources.HOLDOUT, + cmab_uuid=None + ) + + mock_holdout_result = { + 'decision': mock_holdout_decision, + 'error': False, + 'reasons': [] + } + + # Mock get_holdouts_for_flag to return holdouts so the holdout path is taken + with mock.patch.object( + self.config_with_holdouts, + 'get_holdouts_for_flag', + return_value=[holdout] + ): + with mock.patch.object( + self.opt_obj.decision_service, + 'get_variation_for_holdout', + return_value=mock_holdout_result + ): + decision_result = self.opt_obj.decision_service.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context + ) + + self.assertIsNotNone(decision_result) + + # Decision should be valid and from holdout + decision = decision_result['decision'] + self.assertEqual(decision.source, enums.DecisionSources.HOLDOUT) + self.assertIsNotNone(decision.variation) + + def test_no_holdout_decision_falls_through_to_experiment_and_rollout(self): + """When holdout returns no decision, should fall through to experiment and rollout evaluation.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + # Use a user ID that won't be bucketed into holdout + user_context = self.opt_obj.create_user_context('non_holdout_user', {}) + + decision_result = self.decision_service_with_holdouts.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context + ) + + # Should still get a valid decision result + self.assertIsNotNone(decision_result) + self.assertIn('decision', decision_result) + self.assertIn('reasons', decision_result) + + def test_holdout_respects_decision_options(self): + """Should respect decision options when evaluating holdouts.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + # Test with INCLUDE_REASONS option + decision_result = self.decision_service_with_holdouts.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context, + [OptimizelyDecideOption.INCLUDE_REASONS] + ) + + self.assertIsNotNone(decision_result) + self.assertIsInstance(decision_result.get('reasons'), list) + + # Holdout priority and evaluation order tests + + def test_evaluates_holdouts_before_experiments(self): + """Should evaluate holdouts before experiments.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + decision_result = self.decision_service_with_holdouts.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context + ) + + self.assertIsNotNone(decision_result) + + def test_evaluates_global_holdouts_for_all_flags(self): + """Should evaluate global holdouts for all flags.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + # Get global holdouts + global_holdouts = [ + h for h in self.config_with_holdouts.holdouts + if not h.includedFlags or len(h.includedFlags) == 0 + ] + + if global_holdouts: + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + def test_respects_included_and_excluded_flags_configuration(self): + """Should respect included and excluded flags configuration.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + + if feature_flag: + # Get holdouts for this flag + holdouts_for_flag = self.config_with_holdouts.get_holdouts_for_flag('test_feature_in_experiment') + + # Should not include holdouts that exclude this flag + excluded_holdout = next((h for h in holdouts_for_flag if h.key == 'excluded_holdout'), None) + self.assertIsNone(excluded_holdout) + + # Holdout logging and error handling tests + + def test_logs_when_holdout_evaluation_starts(self): + """Should log when holdout evaluation starts.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + # Verify that logger was called + self.assertGreater(self.spy_logger.debug.call_count + self.spy_logger.info.call_count, 0) + + def test_handles_missing_holdout_configuration_gracefully(self): + """Should handle missing holdout configuration gracefully.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + # Temporarily remove holdouts + original_holdouts = self.config_with_holdouts.holdouts + self.config_with_holdouts.holdouts = [] + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + + # Restore original holdouts + self.config_with_holdouts.holdouts = original_holdouts + + def test_handles_invalid_holdout_data_gracefully(self): + """Should handle invalid holdout data gracefully.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + # The method should handle invalid holdout data without crashing + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + # Holdout bucketing behavior tests + + def test_uses_consistent_bucketing_for_same_user(self): + """Should use consistent bucketing for the same user.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_id = 'consistent_user' + user_context1 = self.opt_obj.create_user_context(user_id, {}) + user_context2 = self.opt_obj.create_user_context(user_id, {}) + + result1 = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context1, + [] + ) + + result2 = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context2, + [] + ) + + # Same user should get consistent results + self.assertIsNotNone(result1) + self.assertIsNotNone(result2) + + if result1 and result2: + decision1 = result1[0].get('decision') + decision2 = result2[0].get('decision') + + # If both have decisions, they should match + if decision1 and decision2: + # Handle both dict and Variation entity formats + if decision1.variation: + var1_id = (decision1.variation['id'] if isinstance(decision1.variation, dict) + else decision1.variation.id) + else: + var1_id = None + + if decision2.variation: + var2_id = (decision2.variation['id'] if isinstance(decision2.variation, dict) + else decision2.variation.id) + else: + var2_id = None + + self.assertEqual( + var1_id, var2_id, + "User should get consistent variation across multiple calls" + ) + + def test_uses_bucketing_id_when_provided(self): + """Should use bucketing ID when provided.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_attributes = { + enums.ControlAttributes.BUCKETING_ID: 'custom_bucketing_id' + } + + user_context = self.opt_obj.create_user_context('testUserId', user_attributes) + + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + def test_handles_different_traffic_allocations(self): + """Should handle different traffic allocations.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + # Test with multiple users to see varying bucketing results + users = ['user1', 'user2', 'user3', 'user4', 'user5'] + results = [] + + for user_id in users: + user_context = self.opt_obj.create_user_context(user_id, {}) + result = self.decision_service_with_holdouts.get_variations_for_feature_list( + self.config_with_holdouts, + [feature_flag], + user_context, + [] + ) + results.append(result) + + # All results should be valid + for result in results: + self.assertIsNotNone(result) + self.assertIsInstance(result, list) + + # Holdout integration with feature experiments tests + + def test_checks_holdouts_before_feature_experiments(self): + """Should check holdouts before feature experiments.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('testUserId', {}) + + decision_result = self.decision_service_with_holdouts.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context + ) + + self.assertIsNotNone(decision_result) + + def test_falls_back_to_experiments_if_no_holdout_decision(self): + """Should fall back to experiments if no holdout decision.""" + feature_flag = self.config_with_holdouts.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_context = self.opt_obj.create_user_context('non_holdout_user_123', {}) + + decision_result = self.decision_service_with_holdouts.get_variation_for_feature( + self.config_with_holdouts, + feature_flag, + user_context + ) + + # Should return a valid decision result + self.assertIsNotNone(decision_result) + self.assertIn('decision', decision_result) + self.assertIn('reasons', decision_result) + + # Holdout Impression Events tests + + def test_decide_with_holdout_sends_impression_event(self): + """Should send impression event for holdout decision.""" + # Create optimizely instance with mocked event processor + spy_event_processor = mock.MagicMock() + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'holdout_var_1', + 'key': 'holdout_control', + 'featureEnabled': True, + 'variables': [] + }, + { + 'id': 'holdout_var_2', + 'key': 'holdout_treatment', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'holdout_var_1', + 'endOfRange': 5000 + }, + { + 'entityId': 'holdout_var_2', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt_with_mocked_events = optimizely_module.Optimizely( + datafile=config_json, + logger=self.spy_logger, + error_handler=self.error_handler, + event_processor=spy_event_processor + ) + + try: + # Use a specific user ID that will be bucketed into a holdout + test_user_id = 'user_bucketed_into_holdout' + + config = opt_with_mocked_events.config_manager.get_config() + feature_flag = config.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag, "Feature flag 'test_feature_in_experiment' should exist") + + user_attributes = {} + + user_context = opt_with_mocked_events.create_user_context(test_user_id, user_attributes) + decision = user_context.decide(feature_flag.key) + + self.assertIsNotNone(decision, 'Decision should not be None') + + # Find the actual holdout if this is a holdout decision + actual_holdout = None + if config.holdouts and decision.rule_key: + actual_holdout = next( + (h for h in config.holdouts if h.key == decision.rule_key), + None + ) + + # Only continue if this is a holdout decision + if actual_holdout: + self.assertEqual(decision.flag_key, feature_flag.key) + + holdout_variation = next( + (v for v in actual_holdout.variations if v.get('key') == decision.variation_key), + None + ) + + self.assertIsNotNone( + holdout_variation, + f"Variation '{decision.variation_key}' should be from the chosen holdout '{actual_holdout.key}'" + ) + + self.assertEqual( + decision.enabled, + holdout_variation.get('featureEnabled'), + "Enabled flag should match holdout variation's featureEnabled value" + ) + + # Verify impression event was sent + self.assertGreater(spy_event_processor.process.call_count, 0) + + # Verify impression event contains correct user details + call_args_list = spy_event_processor.process.call_args_list + user_event_found = False + for call_args in call_args_list: + if call_args[0]: # Check positional args + user_event = call_args[0][0] + if hasattr(user_event, 'user_id') and user_event.user_id == test_user_id: + user_event_found = True + break + + self.assertTrue(user_event_found, 'Impression event should contain correct user ID') + finally: + opt_with_mocked_events.close() + + def test_decide_with_holdout_does_not_send_impression_when_disabled(self): + """Should not send impression event when DISABLE_DECISION_EVENT option is used.""" + # Create optimizely instance with mocked event processor + spy_event_processor = mock.MagicMock() + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'holdout_var_1', + 'key': 'holdout_control', + 'featureEnabled': True, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'holdout_var_1', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt_with_mocked_events = optimizely_module.Optimizely( + datafile=config_json, + logger=self.spy_logger, + error_handler=self.error_handler, + event_processor=spy_event_processor + ) + + try: + test_user_id = 'user_bucketed_into_holdout' + + config = opt_with_mocked_events.config_manager.get_config() + feature_flag = config.get_feature_from_key('test_feature_in_experiment') + self.assertIsNotNone(feature_flag) + + user_attributes = {} + + user_context = opt_with_mocked_events.create_user_context(test_user_id, user_attributes) + decision = user_context.decide( + feature_flag.key, + [OptimizelyDecideOption.DISABLE_DECISION_EVENT] + ) + + self.assertIsNotNone(decision, 'Decision should not be None') + + # Find the chosen holdout if this is a holdout decision + chosen_holdout = None + if config.holdouts and decision.rule_key: + chosen_holdout = next( + (h for h in config.holdouts if h.key == decision.rule_key), + None + ) + + if chosen_holdout: + self.assertEqual(decision.flag_key, feature_flag.key) + + # Verify no impression event was sent + spy_event_processor.process.assert_not_called() + finally: + opt_with_mocked_events.close() + + def test_decide_with_holdout_sends_correct_notification_content(self): + """Should send correct notification content for holdout decision.""" + captured_notifications = [] + + def notification_callback(notification_type, user_id, user_attributes, decision_info): + captured_notifications.append(decision_info.copy()) + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'holdout_1', + 'key': 'test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'holdout_var_1', + 'key': 'holdout_control', + 'featureEnabled': True, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'holdout_var_1', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt_with_mocked_events = optimizely_module.Optimizely( + datafile=config_json, + logger=self.spy_logger, + error_handler=self.error_handler + ) + + try: + opt_with_mocked_events.notification_center.add_notification_listener( + enums.NotificationTypes.DECISION, + notification_callback + ) + + test_user_id = 'holdout_test_user' + config = opt_with_mocked_events.config_manager.get_config() + feature_flag = config.get_feature_from_key('test_feature_in_experiment') + holdout = config.holdouts[0] if config.holdouts else None + self.assertIsNotNone(holdout, 'Should have at least one holdout configured') + + holdout_variation = holdout.variations[0] + self.assertIsNotNone(holdout_variation, 'Holdout should have at least one variation') + + mock_experiment = mock.MagicMock() + mock_experiment.key = holdout.key + mock_experiment.id = holdout.id + + # Mock the decision service to return a holdout decision + holdout_decision = decision_service.Decision( + experiment=mock_experiment, + variation=holdout_variation, + source=enums.DecisionSources.HOLDOUT, + cmab_uuid=None + ) + + holdout_decision_result = { + 'decision': holdout_decision, + 'error': False, + 'reasons': [] + } + + # Mock get_variations_for_feature_list to return holdout decision + with mock.patch.object( + opt_with_mocked_events.decision_service, + 'get_variations_for_feature_list', + return_value=[holdout_decision_result] + ): + user_context = opt_with_mocked_events.create_user_context(test_user_id, {}) + decision = user_context.decide(feature_flag.key) + + self.assertIsNotNone(decision, 'Decision should not be None') + self.assertEqual(len(captured_notifications), 1, 'Should have captured exactly one decision notification') + + notification = captured_notifications[0] + rule_key = notification.get('rule_key') + + self.assertEqual(rule_key, holdout.key, 'RuleKey should match holdout key') + + # Verify holdout notification structure + self.assertIn('flag_key', notification, 'Holdout notification should contain flag_key') + self.assertIn('enabled', notification, 'Holdout notification should contain enabled flag') + self.assertIn('variation_key', notification, 'Holdout notification should contain variation_key') + self.assertIn('experiment_id', notification, 'Holdout notification should contain experiment_id') + self.assertIn('variation_id', notification, 'Holdout notification should contain variation_id') + + flag_key = notification.get('flag_key') + self.assertEqual(flag_key, 'test_feature_in_experiment', 'FlagKey should match the requested flag') + + experiment_id = notification.get('experiment_id') + self.assertEqual(experiment_id, holdout.id, 'ExperimentId in notification should match holdout ID') + + variation_id = notification.get('variation_id') + self.assertEqual(variation_id, holdout_variation['id'], 'VariationId should match holdout variation ID') + + variation_key = notification.get('variation_key') + self.assertEqual( + variation_key, + holdout_variation['key'], + 'VariationKey in notification should match holdout variation key' + ) + + enabled = notification.get('enabled') + self.assertIsNotNone(enabled, 'Enabled flag should be present in notification') + self.assertEqual( + enabled, + holdout_variation.get('featureEnabled'), + "Enabled flag should match holdout variation's featureEnabled value" + ) + + self.assertIn(flag_key, config.feature_key_map, f"FlagKey '{flag_key}' should exist in config") + + self.assertIn('variables', notification, 'Notification should contain variables') + self.assertIn('reasons', notification, 'Notification should contain reasons') + self.assertIn( + 'decision_event_dispatched', notification, + 'Notification should contain decision_event_dispatched' + ) + finally: + opt_with_mocked_events.close() + + # DecideAll with holdouts tests (aligned with Swift SDK) + + def test_decide_all_with_global_holdout(self): + """Should apply global holdout to all flags in decide_all.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'global_holdout', + 'key': 'global_test_holdout', + 'status': 'Running', + 'includedFlags': [], # Global - applies to all flags + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'global_holdout_var', + 'key': 'global_holdout_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'global_holdout_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decisions = user_context.decide_all() + + # Global holdout should apply to all flags + self.assertGreater(len(decisions), 0) + + # Check that decisions exist for all flags + for flag_key, decision in decisions.items(): + self.assertIsNotNone(decision) + finally: + opt.close() + + def test_decide_all_with_included_flags(self): + """Should apply holdout only to included flags in decide_all.""" + # Get feature flag IDs + feature1_id = '91111' # test_feature_in_experiment + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'included_holdout', + 'key': 'specific_holdout', + 'status': 'Running', + 'includedFlags': [feature1_id], # Only applies to feature1 + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'included_var', + 'key': 'included_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'included_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decisions = user_context.decide_all() + + self.assertGreater(len(decisions), 0) + + # Verify all flags have decisions + for decision in decisions.values(): + self.assertIsNotNone(decision) + finally: + opt.close() + + def test_decide_all_with_excluded_flags(self): + """Should exclude holdout from excluded flags in decide_all.""" + feature1_id = '91111' # test_feature_in_experiment + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'excluded_holdout', + 'key': 'global_except_one', + 'status': 'Running', + 'includedFlags': [], # Global + 'excludedFlags': [feature1_id], # Except feature1 + 'audienceIds': [], + 'variations': [ + { + 'id': 'excluded_var', + 'key': 'excluded_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'excluded_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decisions = user_context.decide_all() + + # Verify decisions exist for all flags + self.assertGreater(len(decisions), 0) + for decision in decisions.values(): + self.assertIsNotNone(decision) + finally: + opt.close() + + def test_decide_all_with_multiple_holdouts(self): + """Should handle multiple holdouts with correct priority.""" + feature1_id = '91111' + feature2_id = '91112' + + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + # Global holdout (applies to all) + { + 'id': 'global_holdout', + 'key': 'global_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'global_var', + 'key': 'global_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'global_var', + 'endOfRange': 10000 + } + ] + }, + # Specific holdout (only for feature2) + { + 'id': 'specific_holdout', + 'key': 'specific_holdout', + 'status': 'Running', + 'includedFlags': [feature2_id], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'specific_var', + 'key': 'specific_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'specific_var', + 'endOfRange': 10000 + } + ] + }, + # Excluded holdout (all except feature1) + { + 'id': 'excluded_holdout', + 'key': 'excluded_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [feature1_id], + 'audienceIds': [], + 'variations': [ + { + 'id': 'excluded_var', + 'key': 'excluded_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'excluded_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decisions = user_context.decide_all() + + # Verify we got decisions for all flags + self.assertGreater(len(decisions), 0) + finally: + opt.close() + + def test_decide_all_with_enabled_flags_only_option(self): + """Should filter out disabled flags when using enabled_flags_only option.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'disabled_holdout', + 'key': 'disabled_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'disabled_var', + 'key': 'disabled_control', + 'featureEnabled': False, # Feature disabled + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'disabled_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + + # Without enabled_flags_only, all flags should be returned + all_decisions = user_context.decide_all() + + # With enabled_flags_only, disabled flags should be filtered out + enabled_decisions = user_context.decide_all( + options=[OptimizelyDecideOption.ENABLED_FLAGS_ONLY] + ) + + # enabled_decisions should have fewer or equal entries + self.assertLessEqual(len(enabled_decisions), len(all_decisions)) + finally: + opt.close() + + # Impression event metadata tests (aligned with Swift SDK) + + def test_holdout_impression_event_has_correct_metadata(self): + """Should include correct metadata in holdout impression events.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'metadata_holdout', + 'key': 'metadata_test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'metadata_var', + 'key': 'metadata_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'metadata_var', + 'endOfRange': 10000 + } + ] + } + ] + + spy_event_processor = mock.MagicMock() + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely( + datafile=config_json, + event_processor=spy_event_processor + ) + + try: + user_context = opt.create_user_context('test_user', {}) + decision = user_context.decide('test_feature_in_experiment') + + # If this was a holdout decision, verify event metadata + if decision.rule_key == 'metadata_test_holdout': + self.assertGreater(spy_event_processor.process.call_count, 0) + + # Get the user event that was sent + call_args = spy_event_processor.process.call_args_list[0] + user_event = call_args[0][0] + + # Verify user event has correct structure + self.assertIsNotNone(user_event) + finally: + opt.close() + + def test_holdout_impression_respects_send_flag_decisions_false(self): + """Should send holdout impression even when sendFlagDecisions is false.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['sendFlagDecisions'] = False + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'send_flag_holdout', + 'key': 'send_flag_test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'send_flag_var', + 'key': 'send_flag_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'send_flag_var', + 'endOfRange': 10000 + } + ] + } + ] + + spy_event_processor = mock.MagicMock() + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely( + datafile=config_json, + event_processor=spy_event_processor + ) + + try: + user_context = opt.create_user_context('test_user', {}) + decision = user_context.decide('test_feature_in_experiment') + + # Holdout impressions should be sent even when sendFlagDecisions=false + # (unlike rollout impressions) + if decision.rule_key == 'send_flag_test_holdout': + # Verify impression was sent for holdout + self.assertGreater(spy_event_processor.process.call_count, 0) + finally: + opt.close() + + # Holdout status tests (aligned with Swift SDK) + + def test_holdout_not_running_does_not_apply(self): + """Should not apply holdout when status is not Running.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'draft_holdout', + 'key': 'draft_holdout', + 'status': 'Draft', # Not Running + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'draft_var', + 'key': 'draft_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'draft_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decision = user_context.decide('test_feature_in_experiment') + + # Should not be a holdout decision since status is Draft + self.assertNotEqual(decision.rule_key, 'draft_holdout') + finally: + opt.close() + + def test_holdout_concluded_status_does_not_apply(self): + """Should not apply holdout when status is Concluded.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'concluded_holdout', + 'key': 'concluded_holdout', + 'status': 'Concluded', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'concluded_var', + 'key': 'concluded_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'concluded_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decision = user_context.decide('test_feature_in_experiment') + + # Should not be a holdout decision since status is Concluded + self.assertNotEqual(decision.rule_key, 'concluded_holdout') + finally: + opt.close() + + def test_holdout_archived_status_does_not_apply(self): + """Should not apply holdout when status is Archived.""" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'archived_holdout', + 'key': 'archived_holdout', + 'status': 'Archived', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': [], + 'variations': [ + { + 'id': 'archived_var', + 'key': 'archived_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'archived_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + user_context = opt.create_user_context('test_user', {}) + decision = user_context.decide('test_feature_in_experiment') + + # Should not be a holdout decision since status is Archived + self.assertNotEqual(decision.rule_key, 'archived_holdout') + finally: + opt.close() + + # Audience targeting tests for holdouts (aligned with Swift SDK) + + def test_holdout_with_audience_match(self): + """Should bucket user into holdout when audience conditions match.""" + # Using audienceIds that exist in the datafile + # audience '11154' is for "browser_type" = "chrome" + config_dict_with_holdouts = self.config_dict_with_features.copy() + config_dict_with_holdouts['holdouts'] = [ + { + 'id': 'audience_holdout', + 'key': 'audience_test_holdout', + 'status': 'Running', + 'includedFlags': [], + 'excludedFlags': [], + 'audienceIds': ['11154'], # Requires browser_type=chrome + 'variations': [ + { + 'id': 'audience_var', + 'key': 'audience_control', + 'featureEnabled': False, + 'variables': [] + } + ], + 'trafficAllocation': [ + { + 'entityId': 'audience_var', + 'endOfRange': 10000 + } + ] + } + ] + + config_json = json.dumps(config_dict_with_holdouts) + opt = optimizely_module.Optimizely(datafile=config_json) + + try: + # User with matching attribute + user_context_match = opt.create_user_context('test_user', {'browser_type': 'chrome'}) + decision_match = user_context_match.decide('test_feature_in_experiment') + + # User with non-matching attribute + user_context_no_match = opt.create_user_context('test_user', {'browser_type': 'firefox'}) + decision_no_match = user_context_no_match.decide('test_feature_in_experiment') + + # Both should have valid decisions + self.assertIsNotNone(decision_match) + self.assertIsNotNone(decision_no_match) + finally: + opt.close()