# Unique code of this Data Store Server. Not more than 40 characters.
data-store-server-code = DSS2

data-folder = targets/playground/data

# The root directory of the data store
storeroot-dir = ${data-folder}/store

# The directory where the command queue file is located; defaults to storeroot-dir 
commandqueue-dir =

# Port
port = 8890

use-ssl = false

# Session timeout in minutes
session-timeout = 720

# Path to the keystore
keystore.path = ../datastore_server/dist/etc/openBIS.keystore

# Password of the keystore
keystore.password = changeit

# Key password of the keystore
keystore.key-password = changeit

# The check interval (in seconds)
check-interval = 5

# The time-out for clean up work in the shutdown sequence (in seconds).
# Note that that the maximal time for the shutdown sequence to complete can be as large 
# as twice this time.
# Remark: On a network file system, it is not recommended to turn this value to something 
# lower than 180.
shutdown-timeout = 2

# If free disk space goes below value defined here, a notification email will be sent.
# Value must be specified in kilobytes (1048576 = 1024 * 1024 = 1GB). If no high water mark is
# specified or if value is negative, the system will not be watching.
highwater-mark = -1

# If a data set is successfully registered it sends out an email to the registrator. 
# If this property is not specified, no email is sent to the registrator. This property
# does not affect the mails which are sent, when the data set could not be registered.
notify-successful-registration = false

# The URL of the openBIS server
server-url = http://localhost:8888/openbis

# The username to use when contacting the openBIS server
username = etlserver

# The password to use when contacting the openBIS server
password = etlserver

# The base URL for Web client access.
download-url = https://localhost:${port}

# SMTP properties (must start with 'mail' to be considered). 
mail.smtp.host = file://targets/email
mail.from = datastore_server@localhost
# If this property is set a test e-mail will be sent to the specified address after DSS successfully started-up.
mail.test.address = test@localhost

# ---------------- Database parameters

basysbio-databaseEngineCode = postgresql
basysbio-basicDatabaseName = basysbio
basysbio-databaseKind = dev
basysbio-readOnlyGroup = basysbio_readonly
basysbio-readWriteGroup = basysbio_readwrite
basysbio-scriptFolder = source/sql

# Define names of data sources
data-sources = data-source 
data-source.databaseEngineCode = postgresql
data-source.basicDatabaseName = basysbio
data-source.databaseKind = dev


# ---------------- Timing parameters for file system operations on remote shares.

# Time (in seconds) to wait for any file system operation to finish. Operations exceeding this 
# timeout will be terminated. 
timeout = 60
# Number of times that a timed out operation will be tried again (0 means: every file system 
# operation will only ever be performed once).
max-retries = 11
# Time (in seconds) to wait after an operation has been timed out before re-trying.  
failure-interval = 10 

# The period of no write access that needs to pass before an incoming data item is considered 
# complete and ready to be processed (in seconds) [default: 300]. 
# Valid only when auto-detection method is used to determine if an incoming data are ready to be processed.
quiet-period = 10

#maintenance-plugins = post-registration-upload
maintenance-plugins = data-set-clean-up

data-set-clean-up.class = ch.systemsx.cisd.etlserver.plugins.DataSetDeletionMaintenanceTask
data-set-clean-up.interval = 300
data-set-clean-up.data-source = data-source

post-registration-upload.class = eu.basysbio.cisd.dss.PostRegistrationDatabaseUploadTask
post-registration-upload.interval = 1440000
post-registration-upload.database.databaseEngineCode = ${basysbio-databaseEngineCode}
post-registration-upload.database.basicDatabaseName = ${basysbio-basicDatabaseName}
post-registration-upload.database.databaseKind = ${basysbio-databaseKind}
post-registration-upload.database.readOnlyGroup = ${basysbio-readOnlyGroup}
post-registration-upload.database.readWriteGroup = ${basysbio-readWriteGroup}
post-registration-upload.database.scriptFolder = ${basysbio-scriptFolder}

#####################################################################################################
# Data Set Validators

data-set-validators = ChIP-chip-validator, time-series-validator, lca-mtp-time-series-validator, \
               LCA-MTP-PCAV-time-series-validator, LCA-MIC-time-series-validator

####### ChIP-chip Validator #########################################################################

ChIP-chip-validator.data-set-type = CHIP_CHIP
ChIP-chip-validator.path-patterns = *
ChIP-chip-validator.exclude-path-patterns = request.properties
ChIP-chip-validator.columns = transcription-factor, gene-name, gene-function, array-design, \
               microarray-id, chip-peak-position, chip-local-height, chip-score, intergenic, \
               nearby-gene, bsu-number, distance-from-start
ChIP-chip-validator.transcription-factor.header-pattern = TfBSU 
ChIP-chip-validator.transcription-factor.order = 1
ChIP-chip-validator.transcription-factor.value-type = string
ChIP-chip-validator.transcription-factor.value-pattern = ${bsu-number-pattern}

ChIP-chip-validator.gene-name.header-pattern = GeneName/HumanReadable
ChIP-chip-validator.gene-name.order = 2
ChIP-chip-validator.gene-name.value-type = any

ChIP-chip-validator.gene-function.header-pattern = GeneFunction/HumanReadable
ChIP-chip-validator.gene-function.order = 3
ChIP-chip-validator.gene-function.value-type = any

ChIP-chip-validator.array-design.header-pattern = ArrayDesign
ChIP-chip-validator.array-design.order = 4
ChIP-chip-validator.array-design.value-type = string
ChIP-chip-validator.array-design.value-pattern = 070910_BaSysBio_expr|090325_CBS_DTU_Bsu_T2

ChIP-chip-validator.microarray-id.header-pattern = MicroarrayID
ChIP-chip-validator.microarray-id.order = 5
ChIP-chip-validator.microarray-id.value-type = numeric
ChIP-chip-validator.microarray-id.value-range = (0,Infinity)

ChIP-chip-validator.chip-peak-position.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
ChIP-chip-validator.chip-peak-position.header-validator. = ChIP-chip-validator.data-column-header-validator.
ChIP-chip-validator.chip-peak-position.header-validator.data-set-type.terms = ChipPeakPosition[GBAL009126v3]
ChIP-chip-validator.chip-peak-position.order = 6
ChIP-chip-validator.chip-peak-position.value-type = numeric
ChIP-chip-validator.chip-peak-position.value-range = (0,Infinity)

ChIP-chip-validator.chip-local-height.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
ChIP-chip-validator.chip-local-height.header-validator. = ChIP-chip-validator.data-column-header-validator.
ChIP-chip-validator.chip-local-height.header-validator.data-set-type.terms = ChipLocalHeight
ChIP-chip-validator.chip-local-height.order = 7
ChIP-chip-validator.chip-local-height.value-type = numeric
ChIP-chip-validator.chip-local-height.value-range = (0,Infinity)

ChIP-chip-validator.chip-score.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
ChIP-chip-validator.chip-score.header-validator. = ChIP-chip-validator.data-column-header-validator.
ChIP-chip-validator.chip-score.header-validator.data-set-type.terms = ChipScore
ChIP-chip-validator.chip-score.order = 8
ChIP-chip-validator.chip-score.value-type = numeric

ChIP-chip-validator.data-column-header-validator. = default-data-colum-header-validator.
ChIP-chip-validator.data-column-header-validator.elements = ${default-data-colum-header-validator.elements}, growth-phase, genotype
ChIP-chip-validator.data-column-header-validator.experiment-code.terms = \
              LB, LB_Gl, LB_Ma, LB_Gl_Ma, M9, M9_Gl, M9_Ma, M9_Gl_Ma
ChIP-chip-validator.data-column-header-validator.cultivation-method.terms = BR, SF
ChIP-chip-validator.data-column-header-validator.time-point.type = string
ChIP-chip-validator.data-column-header-validator.time-point.pattern = NT
ChIP-chip-validator.data-column-header-validator.cel-loc.terms = CE
ChIP-chip-validator.data-column-header-validator.data-set-type.terms = 
ChIP-chip-validator.data-column-header-validator.value-type.pattern = Value\\[\\]
ChIP-chip-validator.data-column-header-validator.bi-id.pattern = NB
ChIP-chip-validator.data-column-header-validator.cg.pattern = NC
ChIP-chip-validator.data-column-header-validator.growth-phase.type = vocabulary
ChIP-chip-validator.data-column-header-validator.growth-phase.terms = lat, exp, trans, stat
ChIP-chip-validator.data-column-header-validator.genotype.type = string
ChIP-chip-validator.data-column-header-validator.genotype.pattern = .*

ChIP-chip-validator.intergenic.header-pattern = Intergenic\\[GBAL009126v3\\]
ChIP-chip-validator.intergenic.order = 9
ChIP-chip-validator.intergenic.value-type = string
ChIP-chip-validator.intergenic.value-pattern = FALSE|TRUE

ChIP-chip-validator.nearby-gene.header-pattern = NearbyGene\\[GBAL009126v3\\]
ChIP-chip-validator.nearby-gene.order = 10
ChIP-chip-validator.nearby-gene.value-type = any

ChIP-chip-validator.bsu-number.header-pattern = BsuNumber\\[GBAL009126v3\\]
ChIP-chip-validator.bsu-number.order = 11
ChIP-chip-validator.bsu-number.value-type = string
ChIP-chip-validator.bsu-number.value-pattern = NA|${bsu-number-pattern}(;${bsu-number-pattern})*

ChIP-chip-validator.distance-from-start.header-pattern = DistanceFromStart\\[GBAL009126v3\\]
ChIP-chip-validator.distance-from-start.order = 12
ChIP-chip-validator.distance-from-start.value-type = string
ChIP-chip-validator.distance-from-start.value-pattern = NA|(-?[0-9]+(;-?[0-9]+)*)

####### LCA MIC Time Series Validator ###############################################################

LCA-MIC-time-series-validator.data-set-type = LCA_MIC_TIME_SERIES
LCA-MIC-time-series-validator.path-patterns = *
LCA-MIC-time-series-validator.exclude-path-patterns = request.properties
LCA-MIC-time-series-validator.columns = id, data-column
LCA-MIC-time-series-validator.id.header-pattern = BBA ID
LCA-MIC-time-series-validator.id.mandatory = true
LCA-MIC-time-series-validator.id.order = 1
LCA-MIC-time-series-validator.id.value-type = unique
LCA-MIC-time-series-validator.id.value-pattern = BBA[0-9]{4}#[A,B,C]_S20[0-9]{2}[0,1][0-9][0,1,2,3][0-9]-[1-8]
LCA-MIC-time-series-validator.data-column.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
LCA-MIC-time-series-validator.data-column.header-validator. = default-data-colum-header-validator.
LCA-MIC-time-series-validator.data-column.header-validator.data-set-type.terms = LcaMicCfd, LcaMicAbsFl 
LCA-MIC-time-series-validator.data-column.header-validator.value-type.pattern = Value\\[um\\]|Mean\\[Au\\]|Std\\[Au\\]  
LCA-MIC-time-series-validator.data-column.header-validator.bi-id.pattern = NB  
LCA-MIC-time-series-validator.data-column.header-validator.cg.pattern = NC  
LCA-MIC-time-series-validator.data-column.can-define-multiple-columns = true
LCA-MIC-time-series-validator.data-column.value-type = numeric
LCA-MIC-time-series-validator.data-column.allow-empty-values = true
LCA-MIC-time-series-validator.data-column.empty-value-synonyms = #N/A, N/A, NA

####### LCA MTP PCAV Time Series Validator ##########################################################

LCA-MTP-PCAV-time-series-validator.data-set-type = LCA_MTP_PCAV_TIME_SERIES
LCA-MTP-PCAV-time-series-validator.path-patterns = *
LCA-MTP-PCAV-time-series-validator.exclude-path-patterns = request.properties
LCA-MTP-PCAV-time-series-validator.columns = id, no-of-replicates, confidence-level, data-column
LCA-MTP-PCAV-time-series-validator.id.header-pattern = BSB_ID
LCA-MTP-PCAV-time-series-validator.id.mandatory = true
LCA-MTP-PCAV-time-series-validator.id.order = 1
LCA-MTP-PCAV-time-series-validator.id.value-type = unique
LCA-MTP-PCAV-time-series-validator.id.value-pattern = BBA[0-9]+
LCA-MTP-PCAV-time-series-validator.no-of-replicates.header-pattern = no of replicates
LCA-MTP-PCAV-time-series-validator.no-of-replicates.order = 2
LCA-MTP-PCAV-time-series-validator.no-of-replicates.value-type = string
LCA-MTP-PCAV-time-series-validator.no-of-replicates.value-pattern = [0-9]+
LCA-MTP-PCAV-time-series-validator.confidence-level.header-pattern = confidence level
LCA-MTP-PCAV-time-series-validator.confidence-level.order = 3
LCA-MTP-PCAV-time-series-validator.confidence-level.value-type = string
LCA-MTP-PCAV-time-series-validator.confidence-level.value-pattern = [A-Z]
LCA-MTP-PCAV-time-series-validator.data-column.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
LCA-MTP-PCAV-time-series-validator.data-column.header-validator. = default-data-colum-header-validator.
LCA-MTP-PCAV-time-series-validator.data-column.header-validator.biological-replicate-code.pattern = BN 
LCA-MTP-PCAV-time-series-validator.data-column.header-validator.data-set-type.terms = LcaMtpPaCav 
LCA-MTP-PCAV-time-series-validator.data-column.header-validator.value-type.pattern = Average\\[U\\]  
LCA-MTP-PCAV-time-series-validator.data-column.header-validator.bi-id.pattern = NB  
LCA-MTP-PCAV-time-series-validator.data-column.header-validator.cg.pattern = NC  
LCA-MTP-PCAV-time-series-validator.data-column.can-define-multiple-columns = true
LCA-MTP-PCAV-time-series-validator.data-column.value-type = numeric

####### LCA MTP Time Series Validator ###############################################################

lca-mtp-data-set-types = LcaMicCFD, LcaMicabsFl, LcaMtpOD600, LcaMtpAbs900, LcaMtpAbs977, LcaMtpGFP, LcaMtpPa

lca-mtp-time-series-validator.data-set-type = LCA_MTP_TIME_SERIES
lca-mtp-time-series-validator.path-patterns = *
lca-mtp-time-series-validator.exclude-path-patterns = request.properties
lca-mtp-time-series-validator.columns = id, bsb, data-column 
# Identifier column 
lca-mtp-time-series-validator.id.header-pattern = LCAMtpID
lca-mtp-time-series-validator.id.mandatory = true
lca-mtp-time-series-validator.id.order = 1
lca-mtp-time-series-validator.id.value-type = unique
lca-mtp-time-series-validator.id.value-pattern = ((BBA[0-9]{4}#[A-Z])|BSB168|Empty|(FLUO(100|010|001)))_P[0-9]{2}-[0-9]{2}-[0-9]{2}U[1-8](ID|FP|NG)-[A-H](0[1-9]|1(0|1|2)) 
# BSB identifier column 
lca-mtp-time-series-validator.bsb.header-pattern = BSB_ID
lca-mtp-time-series-validator.bsb.order = 2
lca-mtp-time-series-validator.bsb.value-type = string
lca-mtp-time-series-validator.bsb.value-pattern = BBA[0-9]{4}|BSB168|Empty|FLUO100|FLUO010|FLUO001
# Data columns
lca-mtp-time-series-validator.data-column.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
lca-mtp-time-series-validator.data-column.header-validator. = default-data-colum-header-validator.
lca-mtp-time-series-validator.data-column.header-validator.data-set-type.terms = ${lca-mtp-data-set-types}
# LcaMtpOD and LcaMtpGFP should be positive
lca-mtp-time-series-validator.data-column.value-validator = ch.systemsx.cisd.etlserver.validation.HeaderBasedValueValidatorFactory
lca-mtp-time-series-validator.data-column.header-types = positive, real
lca-mtp-time-series-validator.data-column.positive.header-pattern = .*LcaMtpOD.*|.*LcaMtpGFP.*
lca-mtp-time-series-validator.data-column.positive.value-type = numeric
lca-mtp-time-series-validator.data-column.positive.value-range = [0,Infinity) 
# Other values should be real
lca-mtp-time-series-validator.data-column.real.header-pattern = .*
lca-mtp-time-series-validator.data-column.real.value-type = numeric
lca-mtp-time-series-validator.data-column.real.value-range = (-Infinity,Infinity) 
# Many data columns can be defined
lca-mtp-time-series-validator.data-column.can-define-multiple-columns = true


####### Time Series Validator #######################################################################

time-series-validator.data-set-type = TIME_SERIES
time-series-validator.path-patterns = *
time-series-validator.exclude-path-patterns = request.properties
time-series-validator.columns = id, human-readable, controlled-gene, data-column
time-series-validator.id.header-pattern = CompoundID|GeneLocus|Abs
time-series-validator.id.mandatory = true
time-series-validator.id.order = 1
time-series-validator.id.value-validator = ch.systemsx.cisd.etlserver.validation.HeaderBasedValueValidatorFactory
time-series-validator.id.header-types = compound, gene-locus, abs
time-series-validator.id.compound.header-pattern = CompoundID
time-series-validator.id.compound.value-type = unique
time-series-validator.id.compound.value-pattern = BSBME:[0-9]+(BSBME:[0-9])*|CHEBI:[0-9]+(_CHEBI:[0-9]+)*
time-series-validator.id.gene-locus.header-pattern = GeneLocus
time-series-validator.id.gene-locus.value-type = unique
time-series-validator.id.gene-locus.value-pattern = ${bsu-number-pattern}|VMG_[0-9]+_[0-9]+|VMG_[0-9]+_[0-9]+_c
time-series-validator.id.abs.header-pattern = Abs
time-series-validator.id.abs.value-type = unique
time-series-validator.id.abs.value-pattern = OD600
time-series-validator.human-readable.header-pattern = HumanReadable
time-series-validator.human-readable.order = 2
time-series-validator.controlled-gene.header-pattern = ControlledGene
time-series-validator.controlled-gene.order = 3
time-series-validator.data-column.header-validator = eu.basysbio.cisd.dss.DataColumnHeaderValidator
time-series-validator.data-column.header-validator. = default-data-colum-header-validator.
time-series-validator.data-column.can-define-multiple-columns = true
time-series-validator.data-column.value-type = numeric


####### Default Data Column Header Validator ########################################################
default-data-colum-header-validator.elements = experiment-code, cultivation-method, biological-replicate-code, \
			time-point, time-point-type, technical-replicate-code, cel-loc, data-set-type, value-type, scale, bi-id, cg
default-data-colum-header-validator.experiment-code.type = vocabulary 
default-data-colum-header-validator.experiment-code.terms = Ma, Gl, GM, MG 
default-data-colum-header-validator.cultivation-method.type = vocabulary 
default-data-colum-header-validator.cultivation-method.terms = BR, SF, MW, MS 
default-data-colum-header-validator.biological-replicate-code.type = string 
default-data-colum-header-validator.biological-replicate-code.pattern = B[0-9]+(_B[0-9]+)* 
default-data-colum-header-validator.time-point.type = integer 
default-data-colum-header-validator.time-point-type.type = vocabulary 
default-data-colum-header-validator.time-point-type.terms = EX, IN, SI 
default-data-colum-header-validator.technical-replicate-code.type = string 
default-data-colum-header-validator.technical-replicate-code.pattern = T[0-9]+(_T[0-9]+)* 
default-data-colum-header-validator.cel-loc.type = vocabulary 
default-data-colum-header-validator.cel-loc.terms = CE, ES, ME, CY, NC 
default-data-colum-header-validator.data-set-type.type = vocabulary 
default-data-colum-header-validator.data-set-type.terms = ${time-point-data-set-types} 
default-data-colum-header-validator.value-type.type = string  
default-data-colum-header-validator.value-type.pattern =  (Value|Mean|Median|Std|Var|Error|Iqr)\\[(mM|uM|RatioT1|RatioCs||| |\\%)\\]    
default-data-colum-header-validator.scale.type = vocabulary 
default-data-colum-header-validator.scale.terms = LIN, Lin, Log2, Log10, Ln 
default-data-colum-header-validator.bi-id.type = string  
default-data-colum-header-validator.bi-id.pattern = NB|BBA[0-9]*  
default-data-colum-header-validator.cg.type = string  
default-data-colum-header-validator.cg.pattern = NC|${bsu-number-pattern}_(3|5)  

bsu-number-pattern = (BSU(_(misc_|r|t)RNA_)?[0-9]+)

#### End of Data Set Validators #####################################################################

# Comma separated names of processing threads. Each thread should have configuration properties prefixed with its name.
# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
inputs = main-thread

# ---------------------------------------------------------------------------
# main thread configuration
# ---------------------------------------------------------------------------
# The directory to watch for incoming data.
main-thread.incoming-dir = ${data-folder}/incoming
main-thread.incoming-data-completeness-condition = auto-detection
main-thread.delete-unidentified = true
main-thread.data-set-info-extractor = eu.basysbio.cisd.dss.DataSetInfoExtractor
main-thread.type-extractor = ch.systemsx.cisd.etlserver.cifex.CifexTypeExtractor
main-thread.storage-processor = ch.systemsx.cisd.etlserver.CifexStorageProcessor
main-thread.storage-processor.keep-file-regex = .*(?<!request\.properties)$ 
main-thread.storage-processor.processor = eu.basysbio.cisd.dss.StorageProcessor
main-thread.storage-processor.processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
main-thread.storage-processor.processor.time-series-data-set-drop-box-path = ${main-thread.incoming-dir}
main-thread.storage-processor.processor.time-point-data-set-drop-box-path = ${main-thread.dataset-handler.incoming-dir}
main-thread.storage-processor.processor.time-point-data-set-file-name-separator = .
main-thread.storage-processor.processor.data-set-properties-file-name = data-set.properties
main-thread.storage-processor.processor.translation.data-set-types = ${time-point-data-set-types}, LcaMtpPaCav
main-thread.storage-processor.processor.translation.AgilentRNA2color = AGILENT_RNA_2_COLOR
main-thread.storage-processor.processor.translation.MetaboliteLCMS = METABOLITE_LCMS
main-thread.storage-processor.processor.translation.LcaMtpPaCav = LCA_MTP_PA_CAV
main-thread.storage-processor.processor.database.databaseEngineCode = ${basysbio-databaseEngineCode}
main-thread.storage-processor.processor.database.basicDatabaseName = ${basysbio-basicDatabaseName}
main-thread.storage-processor.processor.database.databaseKind = ${basysbio-databaseKind}
main-thread.storage-processor.processor.database.readOnlyGroup = ${basysbio-readOnlyGroup}
main-thread.storage-processor.processor.database.readWriteGroup = ${basysbio-readWriteGroup}
main-thread.storage-processor.processor.database.scriptFolder = ${basysbio-scriptFolder}
time-point-data-set-types = AgilentRNA2color, MetaboliteLCMS

main-thread.dataset-handler = eu.basysbio.cisd.dss.TimeSeriesAndTimePointDataSetHandler
main-thread.dataset-handler.incoming-dir = ${data-folder}/incoming-time-point
main-thread.dataset-handler.incoming-data-completeness-condition = auto-detection
main-thread.dataset-handler.data-set-info-extractor = ch.systemsx.cisd.etlserver.DefaultDataSetInfoExtractor
main-thread.dataset-handler.data-set-info-extractor.strip-file-extension = true
main-thread.dataset-handler.data-set-info-extractor.entity-separator = ${main-thread.storage-processor.processor.time-point-data-set-file-name-separator}
main-thread.dataset-handler.data-set-info-extractor.index-of-sample-code = 0
main-thread.dataset-handler.data-set-info-extractor.space-code = TEST
main-thread.dataset-handler.data-set-info-extractor.data-set-properties-file-name = data-set.properties
main-thread.dataset-handler.type-extractor = eu.basysbio.cisd.dss.TypeExtractor
main-thread.dataset-handler.type-extractor.file-format-type = PROPRIETARY
main-thread.dataset-handler.type-extractor.locator-type = RELATIVE_LOCATION
main-thread.dataset-handler.type-extractor.is-measured = true
main-thread.dataset-handler.storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor