Index

_

__init__() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.registry.StageRegistry method) (yt_framework.core.stage.BaseStage method) (yt_framework.utils.ignore.YTIgnoreMatcher method) (yt_framework.utils.ignore.YTIgnorePattern method) (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method) (ytjobs.mapper.BatchMapper method) (ytjobs.s3.client.S3Client method)

A

add_checkpoint() (in module yt_framework.operations.dependencies)

add_stage() (yt_framework.core.registry.StageRegistry method)

B

BasePipeline (class in yt_framework.core.pipeline) BaseStage (class in yt_framework.core.stage) BatchMapper (class in ytjobs.mapper) boto_config (ytjobs.s3.client.S3ClientOptions attribute) build_code_locally() (in module yt_framework.operations.upload) build_environment() (in module yt_framework.operations.common)

build_map_dependencies() (in module yt_framework.operations.dependencies) build_operation_environment() (in module yt_framework.operations.common) build_pipeline_cli_parser() (in module yt_framework.core.pipeline_cli) build_stage_dependencies() (in module yt_framework.operations.dependencies) build_vanilla_dependencies() (in module yt_framework.operations.dependencies) build_ytjobs_dependencies() (in module yt_framework.operations.dependencies)

C

collect_passthrough_kwargs() (in module yt_framework.operations.common) ColoredFormatter (class in yt_framework.utils.logging) COLORS (yt_framework.utils.logging.ColoredFormatter attribute) command (yt_framework.operations.command_ops.map.MapOperationData attribute), [1] (yt_framework.operations.command_ops.vanilla.VanillaOperationData attribute), [1] config (yt_framework.contracts.stage.StageContext attribute) configs_dir (yt_framework.contracts.stage.StageDependencies property) (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1]

context (yt_framework.core.stage.BaseStage property) create() (ytjobs.s3.client.S3Client static method) create_code_archive() (in module yt_framework.operations.upload) create_path() (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method) create_stage_dependencies() (yt_framework.core.pipeline.BasePipeline method) create_yt_client() (in module yt_framework.yt.factory)

D

DefaultPipeline (class in yt_framework.core.pipeline) delete_checkpoint() (in module ytjobs.checkpoint) dependencies (yt_framework.operations.command_ops.map.MapOperationData attribute), [1] (yt_framework.operations.command_ops.vanilla.VanillaOperationData attribute), [1] deps (yt_framework.contracts.stage.StageContext attribute) discover_stages() (in module yt_framework.core.discovery)

docker_auth (yt_framework.operations.command_ops.map.MapOperationData attribute), [1] (yt_framework.operations.command_ops.vanilla.VanillaOperationData attribute), [1] docker_auth_from_op_config() (in module yt_framework.operations.common) download() (ytjobs.s3.client.S3Client method) download_by_uri() (ytjobs.s3.client.S3Client method) download_table() (in module yt_framework.operations.table)

E

enabled_stage_names() (in module yt_framework.core.pipeline_config) environment (yt_framework.operations.command_ops.map.MapOperationData attribute), [1] (yt_framework.operations.command_ops.vanilla.VanillaOperationData attribute), [1] exists() (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method) (ytjobs.s3.client.S3Client method)

extract_max_failed_jobs() (in module yt_framework.operations.common) extract_operation_resources() (in module yt_framework.operations.common) extract_secure_env_client_kwargs() (in module yt_framework.operations.common)

F

fork() (yt_framework.contracts.stage.StageContext method)

format() (yt_framework.utils.logging.ColoredFormatter method)

G

get_all_stages() (yt_framework.core.registry.StageRegistry method) get_checkpoint_path() (in module ytjobs.checkpoint) get_config_path() (in module ytjobs.config)

get_logger() (in module ytjobs.logging) get_row_count() (in module yt_framework.operations.table) get_stage() (yt_framework.core.registry.StageRegistry method)

H

has_stage() (yt_framework.core.registry.StageRegistry method)

I

ICONS (yt_framework.utils.logging.ColoredFormatter attribute)

init_checkpoint_directory() (in module yt_framework.operations.checkpoint) init_tokenizer_artifact_directory() (in module yt_framework.operations._internal.tokenizer_artifact)

L

list_checkpoints() (in module ytjobs.checkpoint) list_files() (ytjobs.s3.client.S3Client method) list_s3_files() (in module yt_framework.operations.s3) load_checkpoint() (in module ytjobs.checkpoint) load_dict_config_or_exit() (in module yt_framework.core.pipeline_cli) load_env_file() (in module yt_framework.utils.env) load_processing_state() (in module ytjobs.checkpoint)

load_secrets() (in module yt_framework.utils.env) log_config() (in module yt_framework.utils.logging) log_header() (in module yt_framework.utils.logging) log_operation() (in module yt_framework.utils.logging) log_success() (in module yt_framework.utils.logging) log_with_extra() (in module ytjobs.logging) logger (yt_framework.contracts.stage.StageContext attribute) (ytjobs.s3.client.S3ClientOptions attribute)

M

main() (yt_framework.core.pipeline.BasePipeline class method) manage_output() (in module ytjobs.logging) map() (ytjobs.mapper.BatchMapper method) (ytjobs.mapper.StreamMapper method) MapOperationData (class in yt_framework.operations.command_ops.map) mapper_path (yt_framework.operations.command_ops.map.MapOperationData attribute), [1] matches() (yt_framework.utils.ignore.YTIgnorePattern method) max_retries (ytjobs.s3.client.S3ClientOptions attribute) module yt_framework.contracts.stage yt_framework.core.dependencies yt_framework.core.discovery yt_framework.core.pipeline yt_framework.core.pipeline_cli yt_framework.core.pipeline_config yt_framework.core.registry yt_framework.core.stage yt_framework.operations._internal.tokenizer_artifact yt_framework.operations.checkpoint yt_framework.operations.command_ops.map yt_framework.operations.command_ops.map_reduce yt_framework.operations.command_ops.sort yt_framework.operations.command_ops.vanilla yt_framework.operations.common yt_framework.operations.dependencies yt_framework.operations.s3 yt_framework.operations.table yt_framework.operations.upload yt_framework.typed_jobs yt_framework.utils.env yt_framework.utils.ignore yt_framework.utils.logging yt_framework.yt.clients.client_dev yt_framework.yt.clients.client_prod yt_framework.yt.factory ytjobs.checkpoint ytjobs.config ytjobs.logging ytjobs.mapper ytjobs.s3.client

N

name (yt_framework.contracts.stage.StageContext attribute) normalize_upload_modules() (in module yt_framework.core.pipeline) (in module yt_framework.core.pipeline_config)

normalize_upload_paths() (in module yt_framework.core.pipeline) (in module yt_framework.core.pipeline_config)

P

parse_s3_uri() (ytjobs.s3.client.S3Client static method) pickling_dict_from_config() (in module yt_framework.core.pipeline_config) pipeline_config (yt_framework.contracts.stage.StageDependencies property) (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1]

PipelineStageDependencies (class in yt_framework.core.dependencies) prepare_docker_auth() (in module yt_framework.operations.common)

R

read_input_rows() (in module ytjobs.mapper) read_pipeline_mode_for_header() (in module yt_framework.core.pipeline_cli) read_table() (in module yt_framework.operations.table) (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method) redirect_stdout_to_stderr() (in module ytjobs.logging) region_name (ytjobs.s3.client.S3ClientOptions attribute) RESET (yt_framework.utils.logging.ColoredFormatter attribute) resolve_pipeline_config_path() (in module yt_framework.core.pipeline_cli) resolve_tokenizer_archive_name() (in module yt_framework.operations._internal.tokenizer_artifact) resolve_tokenizer_artifact_name() (in module yt_framework.operations._internal.tokenizer_artifact) resolve_tokenizer_artifact_yt_path() (in module yt_framework.operations._internal.tokenizer_artifact)

row_count() (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method) run() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.stage.BaseStage method) run_map() (in module yt_framework.operations.command_ops.map) run_map_reduce() (in module yt_framework.operations.command_ops.map_reduce) run_pipeline_instance_or_exit() (in module yt_framework.core.pipeline_cli) run_reduce() (in module yt_framework.operations.command_ops.map_reduce) run_sort() (in module yt_framework.operations.command_ops.sort) run_vanilla() (in module yt_framework.operations.command_ops.vanilla) run_yql() (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method)

S

S3Client (class in ytjobs.s3.client) S3ClientOptions (class in ytjobs.s3.client) save_checkpoint() (in module ytjobs.checkpoint) save_processing_state() (in module ytjobs.checkpoint) save_s3_paths_to_table() (in module yt_framework.operations.s3) script_path (yt_framework.operations.command_ops.vanilla.VanillaOperationData attribute), [1] set_stage_registry() (yt_framework.core.pipeline.BasePipeline method) setup() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.pipeline.DefaultPipeline method) setup_logging() (in module yt_framework.utils.logging)

should_ignore() (yt_framework.utils.ignore.YTIgnoreMatcher method) should_ignore_file() (in module yt_framework.utils.ignore) stage_dir (yt_framework.contracts.stage.StageContext attribute) (yt_framework.core.stage.BaseStage property) StageBootstrapTypedJob (class in yt_framework.typed_jobs) StageContext (class in yt_framework.contracts.stage) StageDependencies (class in yt_framework.contracts.stage) StageRegistry (class in yt_framework.core.registry) StreamMapper (class in ytjobs.mapper) suppress_all_output() (in module ytjobs.logging)

T

timeout (ytjobs.s3.client.S3ClientOptions attribute)

tokenizer_artifact_name_or_raise() (in module yt_framework.operations._internal.tokenizer_artifact)

U

upload() (ytjobs.s3.client.S3Client method) upload_all_code() (in module yt_framework.operations.upload)

upload_code() (yt_framework.core.pipeline.BasePipeline method) upload_code_archive() (in module yt_framework.operations.upload)

V

VanillaOperationData (class in yt_framework.operations.command_ops.vanilla)

verify_tokenizer_path_or_raise() (in module yt_framework.operations._internal.tokenizer_artifact)

W

write_table() (yt_framework.yt.clients.client_dev.YTDevClient method) (yt_framework.yt.clients.client_prod.YTProdClient method)

Y

yt_client (yt_framework.contracts.stage.StageDependencies property) (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1] yt_framework.contracts.stage module yt_framework.core.dependencies module yt_framework.core.discovery module yt_framework.core.pipeline module yt_framework.core.pipeline_cli module yt_framework.core.pipeline_config module yt_framework.core.registry module yt_framework.core.stage module yt_framework.operations._internal.tokenizer_artifact module yt_framework.operations.checkpoint module yt_framework.operations.command_ops.map module yt_framework.operations.command_ops.map_reduce module yt_framework.operations.command_ops.sort module yt_framework.operations.command_ops.vanilla module yt_framework.operations.common module yt_framework.operations.dependencies module

yt_framework.operations.s3 module yt_framework.operations.table module yt_framework.operations.upload module yt_framework.typed_jobs module yt_framework.utils.env module yt_framework.utils.ignore module yt_framework.utils.logging module yt_framework.yt.clients.client_dev module yt_framework.yt.clients.client_prod module yt_framework.yt.factory module yt_mode_from_pipeline_config() (in module yt_framework.core.pipeline_config) YTDevClient (class in yt_framework.yt.clients.client_dev) YTIgnoreMatcher (class in yt_framework.utils.ignore) YTIgnorePattern (class in yt_framework.utils.ignore) ytjobs.checkpoint module ytjobs.config module ytjobs.logging module ytjobs.mapper module ytjobs.s3.client module YTProdClient (class in yt_framework.yt.clients.client_prod)