Index

_

__init__() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.registry.StageRegistry method) (yt_framework.core.stage.BaseStage method) (yt_framework.utils.ignore.YTIgnoreMatcher method) (yt_framework.utils.ignore.YTIgnorePattern method) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) (ytjobs.mapper.BatchMapper method) (ytjobs.s3.client.S3Client method)

A

add_checkpoint() (in module yt_framework.operations.dependencies)

add_stage() (yt_framework.core.registry.StageRegistry method)

B

BasePipeline (class in yt_framework.core.pipeline) BaseStage (class in yt_framework.core.stage) BatchMapper (class in ytjobs.mapper) build_code_locally() (in module yt_framework.operations.upload) build_environment() (in module yt_framework.operations.common)

build_map_dependencies() (in module yt_framework.operations.dependencies) build_operation_environment() (in module yt_framework.operations.common) build_stage_dependencies() (in module yt_framework.operations.dependencies) build_vanilla_dependencies() (in module yt_framework.operations.dependencies) build_ytjobs_dependencies() (in module yt_framework.operations.dependencies)

C

collect_passthrough_kwargs() (in module yt_framework.operations.common) ColoredFormatter (class in yt_framework.utils.logging) COLORS (yt_framework.utils.logging.ColoredFormatter attribute) command (yt_framework.operations.map.MapOperationData attribute), [1] (yt_framework.operations.vanilla.VanillaOperationData attribute), [1] config (yt_framework.core.stage.StageContext attribute), [1] configs_dir (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1] (yt_framework.core.dependencies.StageDependencies property)

context (yt_framework.core.stage.BaseStage property) create() (ytjobs.s3.client.S3Client static method) create_code_archive() (in module yt_framework.operations.upload) create_path() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) create_stage_dependencies() (yt_framework.core.pipeline.BasePipeline method) create_yt_client() (in module yt_framework.yt.factory)

D

DefaultPipeline (class in yt_framework.core.pipeline) delete_checkpoint() (in module ytjobs.checkpoint) dependencies (yt_framework.operations.map.MapOperationData attribute), [1] (yt_framework.operations.vanilla.VanillaOperationData attribute), [1] deps (yt_framework.core.stage.StageContext attribute), [1] discover_stages() (in module yt_framework.core.discovery)

distinct() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) docker_auth (yt_framework.operations.map.MapOperationData attribute), [1] (yt_framework.operations.vanilla.VanillaOperationData attribute), [1] download() (ytjobs.s3.client.S3Client method) download_by_uri() (ytjobs.s3.client.S3Client method) download_table() (in module yt_framework.operations.table)

E

environment (yt_framework.operations.map.MapOperationData attribute), [1] (yt_framework.operations.vanilla.VanillaOperationData attribute), [1] exists() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) (ytjobs.s3.client.S3Client method)

extract_docker_auth_from_operation_config() (in module yt_framework.operations.common) extract_max_failed_jobs() (in module yt_framework.operations.common) extract_operation_resources() (in module yt_framework.operations.common) extract_secure_env_client_kwargs() (in module yt_framework.operations.common)

F

filter_table() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

fork() (yt_framework.core.stage.StageContext method) format() (yt_framework.utils.logging.ColoredFormatter method)

G

get_all_stages() (yt_framework.core.registry.StageRegistry method) get_checkpoint_path() (in module ytjobs.checkpoint) get_config_path() (in module ytjobs.config) get_logger() (in module ytjobs.logging)

get_row_count() (in module yt_framework.operations.table) get_stage() (yt_framework.core.registry.StageRegistry method) group_by_aggregate() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

H

has_stage() (yt_framework.core.registry.StageRegistry method)

I

ICONS (yt_framework.utils.logging.ColoredFormatter attribute)

init_checkpoint_directory() (in module yt_framework.operations.checkpoint) init_tokenizer_artifact_directory() (in module yt_framework.operations.tokenizer_artifact)

J

join_tables() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

L

limit_table() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) list_checkpoints() (in module ytjobs.checkpoint) list_files() (ytjobs.s3.client.S3Client method) list_s3_files() (in module yt_framework.operations.s3) load_checkpoint() (in module ytjobs.checkpoint) load_env_file() (in module yt_framework.utils.env)

load_processing_state() (in module ytjobs.checkpoint) load_secrets() (in module yt_framework.utils.env) log_config() (in module yt_framework.utils.logging) log_header() (in module yt_framework.utils.logging) log_operation() (in module yt_framework.utils.logging) log_success() (in module yt_framework.utils.logging) log_with_extra() (in module ytjobs.logging) logger (yt_framework.core.stage.StageContext attribute), [1]

M

main() (yt_framework.core.pipeline.BasePipeline class method) manage_output() (in module ytjobs.logging) map() (ytjobs.mapper.BatchMapper method) (ytjobs.mapper.StreamMapper method) MapOperationData (class in yt_framework.operations.map) mapper_path (yt_framework.operations.map.MapOperationData attribute), [1] matches() (yt_framework.utils.ignore.YTIgnorePattern method) module yt_framework.core.dependencies yt_framework.core.discovery yt_framework.core.pipeline yt_framework.core.registry yt_framework.core.stage yt_framework.operations.checkpoint yt_framework.operations.common yt_framework.operations.dependencies yt_framework.operations.map yt_framework.operations.map_reduce yt_framework.operations.s3 yt_framework.operations.sort yt_framework.operations.table yt_framework.operations.tokenizer_artifact yt_framework.operations.upload yt_framework.operations.vanilla yt_framework.typed_jobs yt_framework.utils.env yt_framework.utils.ignore yt_framework.utils.logging yt_framework.yt.client_dev yt_framework.yt.client_prod yt_framework.yt.factory ytjobs.checkpoint ytjobs.config ytjobs.logging ytjobs.mapper ytjobs.s3.client

N

name (yt_framework.core.stage.BaseStage attribute) (yt_framework.core.stage.StageContext attribute), [1]

P

parse_s3_uri() (ytjobs.s3.client.S3Client static method) pipeline_config (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1] (yt_framework.core.dependencies.StageDependencies property)

PipelineStageDependencies (class in yt_framework.core.dependencies) prepare_docker_auth() (in module yt_framework.operations.common)

R

read_input_rows() (in module ytjobs.mapper) read_table() (in module yt_framework.operations.table) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) redirect_stdout_to_stderr() (in module ytjobs.logging) RESET (yt_framework.utils.logging.ColoredFormatter attribute) resolve_tokenizer_archive_name() (in module yt_framework.operations.tokenizer_artifact) resolve_tokenizer_artifact_name() (in module yt_framework.operations.tokenizer_artifact) resolve_tokenizer_artifact_yt_path() (in module yt_framework.operations.tokenizer_artifact) row_count() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) run() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.stage.BaseStage method) run_map() (in module yt_framework.operations.map) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

run_map_reduce() (in module yt_framework.operations.map_reduce) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) run_reduce() (in module yt_framework.operations.map_reduce) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) run_sort() (in module yt_framework.operations.sort) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) run_vanilla() (in module yt_framework.operations.vanilla) (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) run_yql() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

S

S3Client (class in ytjobs.s3.client) save_checkpoint() (in module ytjobs.checkpoint) save_processing_state() (in module ytjobs.checkpoint) save_s3_paths_to_table() (in module yt_framework.operations.s3) script_path (yt_framework.operations.vanilla.VanillaOperationData attribute), [1] select_columns() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) set_stage_registry() (yt_framework.core.pipeline.BasePipeline method) setup() (yt_framework.core.pipeline.BasePipeline method) (yt_framework.core.pipeline.DefaultPipeline method) setup_logging() (in module yt_framework.utils.logging)

should_ignore() (yt_framework.utils.ignore.YTIgnoreMatcher method) should_ignore_file() (in module yt_framework.utils.ignore) sort_table() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) stage_dir (yt_framework.core.stage.BaseStage property) (yt_framework.core.stage.StageContext attribute), [1] StageBootstrapTypedJob (class in yt_framework.typed_jobs) StageContext (class in yt_framework.core.stage) StageDependencies (class in yt_framework.core.dependencies) StageRegistry (class in yt_framework.core.registry) StreamMapper (class in ytjobs.mapper) suppress_all_output() (in module ytjobs.logging)

U

union_tables() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) upload() (ytjobs.s3.client.S3Client method) upload_all_code() (in module yt_framework.operations.upload) upload_code() (yt_framework.core.pipeline.BasePipeline method)

upload_code_archive() (in module yt_framework.operations.upload) upload_directory() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method) upload_file() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

V

VanillaOperationData (class in yt_framework.operations.vanilla)

W

write_table() (yt_framework.yt.client_dev.YTDevClient method) (yt_framework.yt.client_prod.YTProdClient method)

Y

yt_client (yt_framework.core.dependencies.PipelineStageDependencies attribute), [1] (yt_framework.core.dependencies.StageDependencies property) yt_framework.core.dependencies module yt_framework.core.discovery module yt_framework.core.pipeline module yt_framework.core.registry module yt_framework.core.stage module yt_framework.operations.checkpoint module yt_framework.operations.common module yt_framework.operations.dependencies module yt_framework.operations.map module yt_framework.operations.map_reduce module yt_framework.operations.s3 module yt_framework.operations.sort module yt_framework.operations.table module yt_framework.operations.tokenizer_artifact module yt_framework.operations.upload module

yt_framework.operations.vanilla module yt_framework.typed_jobs module yt_framework.utils.env module yt_framework.utils.ignore module yt_framework.utils.logging module yt_framework.yt.client_dev module yt_framework.yt.client_prod module yt_framework.yt.factory module YTDevClient (class in yt_framework.yt.client_dev) YTIgnoreMatcher (class in yt_framework.utils.ignore) YTIgnorePattern (class in yt_framework.utils.ignore) ytjobs.checkpoint module ytjobs.config module ytjobs.logging module ytjobs.mapper module ytjobs.s3.client module YTProdClient (class in yt_framework.yt.client_prod)