Description:

Reads a local file and sends it to an S3 compatible bucket.

Parameters:

Param Name

Is Required

Inputs Type

Input Default Value

Description

amazonS3ConnectionProfile

True

String


The alias for the connection profile needed to connect to Amazon S3

directory

True

String


The directory where the input files should be read from

fileNameFilter

True

String


The name of the files to be read. Can contain a '*' wildcard so that multiple files are read

chunkSize

False

Integer

10000

The number of records to be read before being written

Examples:

jobName: cdpe-bb-instr-enrl-to-aoi
steps:
  - stepName: cdpe-bb-instr-enrl-inbound-to-ih
    stepType: csvToSqlStep
    stepParameters:
      sourceFilePath: /batch/query-manager/incoming/CDPE_BB_INSTR_ENRL*.csv
      destinationConnectionProfile: ih
      destinationTableName: cdpe_bb_instr_enrl
      destinationSchema: mywsu
      createTableIfMissing: true
      dropAndRebuild: true
      chunkSize: "1000"
  - stepType: sqlToCsvStep
    stepName: cdpe-bb-instr-enrl-outbound-to-sftp
    stepParameters:
      sourceConnectionProfile: ih
      selectStatement: select * from mywsu.cdpe_bb_instr_enrl
      destinationFilePath: "'/batch/query-manager/outgoing/CDPE_BB_INSTR_ENRL_' + new java.text.SimpleDateFormat('yyyy_MM_dd_hh_mm_ss').format(new java.util.Date()) + '.csv'"
  - stepType: fileToAmazonS3BucketStep
    stepName: cdpe-bb-instr-enrl-to-aoi-s3-bucket
    stepParameters:
      directory: /batch/query-manager/outgoing
      fileNameFilter: CDPE_BB_INSTR_ENRL_*.csv
      amazonS3ConnectionProfile: aoi
  - stepName: deleteIncomingFile
    stepType: execCommandStep
    stepParameters:
      command: rm /batch/query-manager/incoming/CDPE_BB_INSTR_ENRL*.csv
      workingDirectory: /batch/query-manager/incoming
      simpleCommand: true
  - stepName: deleteOutgoingFiles
    stepType: execCommandStep
    stepParameters:
      command: rm /batch/query-manager/outgoing/CDPE_BB_INSTR_ENRL*.csv
      workingDirectory: /batch/query-manager/incoming
      simpleCommand: true