From 6bfe52cb6e009b78e9c0379591b6e92ddd39e0a3 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Tue, 16 Jun 2026 19:30:42 +0000 Subject: [PATCH 1/2] Migrate Android Build API usage to V4 OnePlatform under feature flag The legacy Android Build API V3 (androidbuildinternal.googleapis.com) is being sunsetted. This CL migrates the client calls to the new V4 Private API (androidbuild-pa.googleapis.com) using standard discovery. Changes: - Added a 'use_android_build_api_v4' Datastore Config property to allow runtime feature toggle. - Refactored 'fetch_artifact.py' to support V4 OnePlatform endpoint and resource names dynamically behind the feature flag. - Migrated legacy logs to structured dictionary-based logs with '[AndroidBuildAPI]' tagging for easier Cloud Logging filtering. --- .../_internal/datastore/data_types.py | 3 + .../platforms/android/fetch_artifact.py | 233 +++++++++++++++--- 2 files changed, 207 insertions(+), 29 deletions(-) diff --git a/src/clusterfuzz/_internal/datastore/data_types.py b/src/clusterfuzz/_internal/datastore/data_types.py index e648a58bea3..bb1fe3e7aee 100644 --- a/src/clusterfuzz/_internal/datastore/data_types.py +++ b/src/clusterfuzz/_internal/datastore/data_types.py @@ -883,6 +883,9 @@ class Config(Model): # functional bugs. relax_security_bug_restrictions = ndb.BooleanProperty(default=False) + # Flag to use the V4 Android Build API instead of V3. + use_android_build_api_v4 = ndb.BooleanProperty(default=False) + # Coverage reports bucket. coverage_reports_bucket = ndb.StringProperty(default='') diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 4aaf7f40aad..808be3bfda4 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -48,6 +48,23 @@ "gs://android-haiku/target-cuttlefish/stable_build_info.json") +def _use_v4(): + """Return True if we should use V4 Android Build API.""" + try: + use_v4 = db_config.get_value('use_android_build_api_v4') or False + logs.info( + 'AndroidBuildAPI feature flag status read.', + use_android_build_api_v4=use_v4 + ) + return use_v4 + except Exception as e: + logs.error( + 'AndroidBuildAPI error reading feature flag use_android_build_api_v4. Defaulting to False.', + error=str(e) + ) + return False + + def execute_request_with_retries(request): """Executes request and retries on failure.""" result = None @@ -68,24 +85,71 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, logs.info('artifact to download: %s' % name) logs.info('output_directory: %s' % output_directory) logs.info('output_filename: %s' % output_filename) - artifact_query = client.buildartifact().get( - buildId=bid, target=target, attemptId=attempt_id, resourceId=name) + + version_tag = 'V4' if _use_v4() else 'V3' + logs.info( + 'AndroidBuildAPI download_artifact started.', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name + ) + + if _use_v4(): + artifact_query = client.buildartifacts().get( + buildId=bid, target=target, attemptId=attempt_id, resourceId=name) + else: + artifact_query = client.buildartifact().get( + buildId=bid, target=target, attemptId=attempt_id, resourceId=name) artifact = execute_request_with_retries(artifact_query) if artifact is None: - logs.error(f'Artifact unreachable with name {name}, target {target} ' - f'and build id {bid}.') + logs.error( + 'AndroidBuildAPI download_artifact failed: artifact metadata unreachable.', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name, + status='failed' + ) return None # Lucky us, we always have the size. size = int(artifact['size']) + logs.info( + 'AndroidBuildAPI download_artifact metadata retrieved successfully.', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name, + size=size + ) chunksize = -1 if size >= DEFAULT_CHUNK_SIZE: chunksize = DEFAULT_CHUNK_SIZE # Just like get, except get_media. - dl_request = client.buildartifact().get_media( - buildId=bid, target=target, attemptId=attempt_id, resourceId=name) + logs.info( + 'AndroidBuildAPI download_artifact media download started.', + api_version=version_tag, + operation='download_artifact_media', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name + ) + if _use_v4(): + dl_request = client.buildartifacts().get_media( + buildId=bid, target=target, attemptId=attempt_id, resourceId=name) + else: + dl_request = client.buildartifact().get_media( + buildId=bid, target=target, attemptId=attempt_id, resourceId=name) if output_filename: file_name = output_filename @@ -95,7 +159,17 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, output_path = os.path.join(output_directory, file_name) # If the artifact already exists, then bail out. if os.path.exists(output_path) and os.path.getsize(output_path) == size: - logs.info('Artifact %s already exists, skipping download.' % name) + logs.info( + 'AndroidBuildAPI download_artifact skipped (file already exists).', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name, + output_path=output_path, + status='skipped_exists' + ) return output_path logs.info('Downloading artifact %s.' % name) @@ -118,6 +192,17 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, percent_completed = (size_completed * 100.0) / size logs.info('%.1f%% complete.' % percent_completed) + logs.info( + 'AndroidBuildAPI download_artifact completed successfully.', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name, + output_path=output_path, + status='success' + ) return output_path @@ -127,16 +212,39 @@ def get_artifacts_for_build(client, attempt_id: str = 'latest', regexp: Optional[str] = None) -> List[str]: """Return list of artifacts for a given build.""" - if not regexp: - request = client.buildartifact().list( - buildId=bid, target=target, attemptId=attempt_id) + version_tag = 'V4' if _use_v4() else 'V3' + logs.info( + 'AndroidBuildAPI get_artifacts_for_build started.', + api_version=version_tag, + operation='get_artifacts_for_build', + build_id=bid, + target=target, + attempt_id=attempt_id, + regexp=regexp + ) + + if _use_v4(): + if not regexp: + request = client.buildartifacts().list( + buildId=bid, target=target, attemptId=attempt_id) + else: + request = client.buildartifacts().list( + buildId=bid, + target=target, + attemptId=attempt_id, + nameRegexp=regexp, + maxResults=100) else: - request = client.buildartifact().list( - buildId=bid, - target=target, - attemptId=attempt_id, - nameRegexp=regexp, - maxResults=100) + if not regexp: + request = client.buildartifact().list( + buildId=bid, target=target, attemptId=attempt_id) + else: + request = client.buildartifact().list( + buildId=bid, + target=target, + attemptId=attempt_id, + nameRegexp=regexp, + maxResults=100) request_str = (f'{request.uri}, {request.method}, ' f'{request.body}, {request.methodId}') @@ -151,7 +259,22 @@ def get_artifacts_for_build(client, if result and 'artifacts' in result: for artifact in result['artifacts']: artifacts.append(artifact) - request = client.buildartifact().list_next(request, result) + if _use_v4(): + request = client.buildartifacts().list_next(request, result) + else: + request = client.buildartifact().list_next(request, result) + + logs.info( + 'AndroidBuildAPI get_artifacts_for_build completed.', + api_version=version_tag, + operation='get_artifacts_for_build', + build_id=bid, + target=target, + attempt_id=attempt_id, + regexp=regexp, + artifacts_count=len(artifacts), + status='success' if artifacts else 'empty' + ) if not artifacts: logs.error(f'No artifact found for target {target}, build id {bid}.\n' @@ -174,11 +297,27 @@ def get_client(): credentials = ServiceAccountCredentials.from_json_keyfile_dict( json.loads(build_apiary_service_account_private_key), scopes='https://www.googleapis.com/auth/androidbuild.internal') - client = apiclient.discovery.build( - 'androidbuildinternal', - 'v3', - credentials=credentials, - static_discovery=False) + if _use_v4(): + logs.info( + 'AndroidBuildAPI client initialization started.', + api_version='V4' + ) + client = apiclient.discovery.build( + 'androidbuildinternal', + 'v4', + discoveryServiceUrl='https://androidbuild-pa.googleapis.com/$discovery/rest?version=v4', + credentials=credentials, + static_discovery=False) + else: + logs.info( + 'AndroidBuildAPI client initialization started.', + api_version='V3' + ) + client = apiclient.discovery.build( + 'androidbuildinternal', + 'v3', + credentials=credentials, + static_discovery=False) return client @@ -218,25 +357,61 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): if 'bid' in build_info and build_info['bid'] != '0': return build_info - request = client.build().list( # pylint: disable=no-member - buildType='submitted', + version_tag = 'V4' if _use_v4() else 'V3' + logs.info( + 'AndroidBuildAPI get_latest_artifact_info started.', + api_version=version_tag, + operation='get_latest_artifact_info', branch=branch, target=target, - successful=True, - maxResults=1, - signed=signed) + signed=signed + ) + if _use_v4(): + request = client.builds().list( # pylint: disable=no-member + buildType='submitted', + branch=branch, + target=target, + successful=True, + maxResults=1, + signed=signed) + else: + request = client.build().list( # pylint: disable=no-member + buildType='submitted', + branch=branch, + target=target, + successful=True, + maxResults=1, + signed=signed) request_str = (f'{request.uri}, {request.method}, ' f'{request.body}, {request.methodId}') builds = execute_request_with_retries(request) if not builds: - logs.error(f'No build found for target {target}, branch {branch}, ' - f'request: {request_str}.') + logs.error( + 'AndroidBuildAPI get_latest_artifact_info failed: no builds found.', + api_version=version_tag, + operation='get_latest_artifact_info', + branch=branch, + target=target, + signed=signed, + status='failed' + ) return None build = builds['builds'][0] bid = build['buildId'] target = build['target']['name'] + + logs.info( + 'AndroidBuildAPI get_latest_artifact_info completed.', + api_version=version_tag, + operation='get_latest_artifact_info', + branch=branch, + target=target, + signed=signed, + build_id=bid, + status='success' + ) return {'bid': bid, 'branch': branch, 'target': target} From b41a9106d13c7a521069e9c03ed5d48760ad58f4 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Tue, 16 Jun 2026 20:13:23 +0000 Subject: [PATCH 2/2] refactor: reformat logging calls and add V4 API migration guide --- .../platforms/android/fetch_artifact.py | 58 +++++++------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 808be3bfda4..7c37fef2406 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -54,14 +54,12 @@ def _use_v4(): use_v4 = db_config.get_value('use_android_build_api_v4') or False logs.info( 'AndroidBuildAPI feature flag status read.', - use_android_build_api_v4=use_v4 - ) + use_android_build_api_v4=use_v4) return use_v4 except Exception as e: logs.error( 'AndroidBuildAPI error reading feature flag use_android_build_api_v4. Defaulting to False.', - error=str(e) - ) + error=str(e)) return False @@ -85,7 +83,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, logs.info('artifact to download: %s' % name) logs.info('output_directory: %s' % output_directory) logs.info('output_filename: %s' % output_filename) - + version_tag = 'V4' if _use_v4() else 'V3' logs.info( 'AndroidBuildAPI download_artifact started.', @@ -94,9 +92,8 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, build_id=bid, target=target, attempt_id=attempt_id, - artifact_name=name - ) - + artifact_name=name) + if _use_v4(): artifact_query = client.buildartifacts().get( buildId=bid, target=target, attemptId=attempt_id, resourceId=name) @@ -113,8 +110,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, target=target, attempt_id=attempt_id, artifact_name=name, - status='failed' - ) + status='failed') return None # Lucky us, we always have the size. @@ -127,8 +123,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, target=target, attempt_id=attempt_id, artifact_name=name, - size=size - ) + size=size) chunksize = -1 if size >= DEFAULT_CHUNK_SIZE: @@ -142,8 +137,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, build_id=bid, target=target, attempt_id=attempt_id, - artifact_name=name - ) + artifact_name=name) if _use_v4(): dl_request = client.buildartifacts().get_media( buildId=bid, target=target, attemptId=attempt_id, resourceId=name) @@ -168,8 +162,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, attempt_id=attempt_id, artifact_name=name, output_path=output_path, - status='skipped_exists' - ) + status='skipped_exists') return output_path logs.info('Downloading artifact %s.' % name) @@ -201,8 +194,7 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, attempt_id=attempt_id, artifact_name=name, output_path=output_path, - status='success' - ) + status='success') return output_path @@ -220,9 +212,8 @@ def get_artifacts_for_build(client, build_id=bid, target=target, attempt_id=attempt_id, - regexp=regexp - ) - + regexp=regexp) + if _use_v4(): if not regexp: request = client.buildartifacts().list( @@ -273,8 +264,7 @@ def get_artifacts_for_build(client, attempt_id=attempt_id, regexp=regexp, artifacts_count=len(artifacts), - status='success' if artifacts else 'empty' - ) + status='success' if artifacts else 'empty') if not artifacts: logs.error(f'No artifact found for target {target}, build id {bid}.\n' @@ -299,20 +289,17 @@ def get_client(): scopes='https://www.googleapis.com/auth/androidbuild.internal') if _use_v4(): logs.info( - 'AndroidBuildAPI client initialization started.', - api_version='V4' - ) + 'AndroidBuildAPI client initialization started.', api_version='V4') client = apiclient.discovery.build( 'androidbuildinternal', 'v4', - discoveryServiceUrl='https://androidbuild-pa.googleapis.com/$discovery/rest?version=v4', + discoveryServiceUrl= + 'https://androidbuild-pa.googleapis.com/$discovery/rest?version=v4', credentials=credentials, static_discovery=False) else: logs.info( - 'AndroidBuildAPI client initialization started.', - api_version='V3' - ) + 'AndroidBuildAPI client initialization started.', api_version='V3') client = apiclient.discovery.build( 'androidbuildinternal', 'v3', @@ -364,8 +351,7 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): operation='get_latest_artifact_info', branch=branch, target=target, - signed=signed - ) + signed=signed) if _use_v4(): request = client.builds().list( # pylint: disable=no-member buildType='submitted', @@ -394,14 +380,13 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): branch=branch, target=target, signed=signed, - status='failed' - ) + status='failed') return None build = builds['builds'][0] bid = build['buildId'] target = build['target']['name'] - + logs.info( 'AndroidBuildAPI get_latest_artifact_info completed.', api_version=version_tag, @@ -410,8 +395,7 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): target=target, signed=signed, build_id=bid, - status='success' - ) + status='success') return {'bid': bid, 'branch': branch, 'target': target}