create table emr_commands (
  emr_cluster ,
  job_name text not null,
  region text not null default 'us-east-1',
  class text not null,
  s3_jar_path text not null,
  spark_conf ,
  spark_packages ,
  args ,
  s3_log_details ,
  load_data ,
  save_results 
);

comment on column emr_commands.emr_cluster is '
  The EMR cluster on which to launch the job (aka step)
';
comment on column emr_commands.job_name is '
  NAme to be assigned to the Job
';
comment on column emr_commands.region is '
  the AWS Region
';
comment on column emr_commands.class is '
  The spark job's `Main.class` to run
';
comment on column emr_commands.s3_jar_path is '
  the s3 path to the jar of the job to be run
';
comment on column emr_commands.spark_conf is '
  options that will be sent to spark as --conf key=value, e.g. `--conf spark.driver.extraJavaOptions=-Denvironment=integration`
';
comment on column emr_commands.spark_packages is '
  External maven dependencies to include with Spark job
';
comment on column emr_commands.args is '
  command line arguments to be passe to the job's main class
';
comment on column emr_commands.s3_log_details is '
  AWS EMR periodically (currently every 5 minutes) zips up all the logs and upload them into S3. This does not work well with sundial's _live logs_. Make use of S3 Logs to visualise in real time job's log on sundial's live logs panel
';
comment on column emr_commands.load_data is '
  will trigger a s3 dist cp job for every entry in this array, can be used to load data from s3 into HDFS. **requires Hadoop** as application on the cluster
';
comment on column emr_commands.save_results is '
  will trigger a s3 dist cp job for every entry in this array, can be used to save data from HDFS to s3. **requires Hadoop** as application on the cluster
';