Skip to content

Commit da27c6a

Browse files
authored
feat: add decoder CLARA engine and a decoderless Clas12Reader I/O service (#1229)
* add convenience method * remove unused stuff * add decoding engine * add it to the clara yaml * use a pool * hmm * allow decoder instances to share ConstantsManagers * share ConstantsManagers * kludge test * cleanup * try this * inherit ConstantsManagers * cleanup * only check tables if not shared * higher ports on macos, cleanup process dpe process * remove ineffective pid trap * Revert "higher ports on macos, cleanup process dpe process" This reverts commit 3c0135e. * Revert "fix job name" This reverts commit a5512a9. * Revert "decouple ubuntu/macos builds to reduce wait" This reverts commit 446a6a5. * Revert "remove unnecessary reader class" This reverts commit 45e96c8. * Revert "remove example engine" This reverts commit d1dce3b. * restore reported data type * rename class * undo * fix rebase oops * rename class * fix rebase oops * cleanup * cleanup * stf * just events, not per thread * reduce constants sharing * bugfix * share one RCDBManager * bugfix * restore Adjust the number of events based on the number of threads. * DecoderEngine: just pass along HIPO events * add Clas12Reader, no decoding * dummy initialization value * bugfix
1 parent dce382e commit da27c6a

5 files changed

Lines changed: 155 additions & 24 deletions

File tree

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToEvioReader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public int readEventCount() throws EventReaderException {
3939

4040
@Override
4141
public ByteOrder readByteOrder() throws EventReaderException {
42-
return reader.getFileByteOrder();
42+
return ByteOrder.LITTLE_ENDIAN; //reader.getFileByteOrder();
4343
}
4444

4545
@Override

‎common-tools/clas-detector/src/main/java/org/jlab/detector/decode/CLASDecoder.java‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515

1616
import org.jlab.io.base.DataEvent;
1717
import org.jlab.io.evio.EvioDataEvent;
18-
import org.jlab.io.hipo.HipoDataEvent;
19-
import org.jlab.io.hipo.HipoDataSync;
2018

2119
import org.jlab.jnp.hipo4.data.Bank;
2220
import org.jlab.jnp.hipo4.data.Event;
@@ -38,27 +36,28 @@ public class CLASDecoder {
3836
protected SchemaFactory schemaFactory = new SchemaFactory();
3937
private CodaEventDecoder codaDecoder = null;
4038
private List<DetectorDataDgtz> dataList = new ArrayList<>();
41-
private HipoDataSync writer = null;
42-
private HipoDataEvent hipoEvent = null;
4339
private boolean isRunNumberFixed = false;
4440
private int decoderDebugMode = 0;
4541
private ModeAHDC ahdcExtractor = new ModeAHDC();
46-
private RCDBManager rcdbManager = new RCDBManager();
42+
private static RCDBManager rcdbManager = new RCDBManager();
4743

4844
public CLASDecoder(boolean development){
4945
codaDecoder = new CodaEventDecoder();
5046
detectorDecoder = new DetectorEventDecoder(development);
51-
writer = new HipoDataSync();
52-
hipoEvent = (HipoDataEvent) writer.createEvent();
5347
String dir = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4");
5448
schemaFactory.initFromDirectory(dir);
5549
}
5650

5751
public CLASDecoder(){
5852
codaDecoder = new CodaEventDecoder();
5953
detectorDecoder = new DetectorEventDecoder();
60-
writer = new HipoDataSync();
61-
hipoEvent = (HipoDataEvent) writer.createEvent();
54+
String dir = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4");
55+
schemaFactory.initFromDirectory(dir);
56+
}
57+
58+
public CLASDecoder(CLASDecoder d) {
59+
codaDecoder = new CodaEventDecoder();
60+
detectorDecoder = new DetectorEventDecoder(d.detectorDecoder);
6261
String dir = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4");
6362
schemaFactory.initFromDirectory(dir);
6463
}
@@ -792,4 +791,8 @@ public Event getDecodedEvent(EvioDataEvent rawEvent, int run, int counter, Doubl
792791

793792
return decodedEvent;
794793
}
794+
795+
public Event getDecodedEvent(EvioDataEvent rawEvent) {
796+
return getDecodedEvent(rawEvent, -1, -1, null, null);
797+
}
795798
}

‎common-tools/clas-detector/src/main/java/org/jlab/detector/decode/DetectorEventDecoder.java‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,17 @@ public DetectorEventDecoder(boolean development){
4545
}
4646
}
4747

48+
public DetectorEventDecoder(){
49+
this.initDecoder();
50+
}
51+
52+
public DetectorEventDecoder(DetectorEventDecoder d) {
53+
translationManager = d.translationManager;
54+
fitterManager = d.fitterManager;
55+
scalerManager = d.scalerManager;
56+
initDecoder(false);
57+
}
58+
4859
public void setTimestamp(String timestamp) {
4960
translationManager.setTimeStamp(timestamp);
5061
fitterManager.setTimeStamp(timestamp);
@@ -80,10 +91,6 @@ public float getRcdbSolenoidScale() {
8091
getValue()).floatValue();
8192
}
8293

83-
public DetectorEventDecoder(){
84-
this.initDecoder();
85-
}
86-
8794
public final void initDecoderDev(){
8895
keysTrans = Arrays.asList(new DetectorType[]{ DetectorType.HTCC,DetectorType.BST,DetectorType.RTPC} );
8996
tablesTrans = Arrays.asList(new String[]{ "/daq/tt/clasdev/htcc","/daq/tt/clasdev/svt","/daq/tt/clasdev/rtpc" });
@@ -95,7 +102,11 @@ public final void initDecoderDev(){
95102
"/runcontrol/helicity","/daq/config/scalers/dsc1"}));
96103
}
97104

98-
public final void initDecoder(){
105+
public final void initDecoder() {
106+
initDecoder(true);
107+
}
108+
109+
public final void initDecoder(boolean initializeManagers){
99110

100111
// Detector translation table
101112
keysTrans = Arrays.asList(new DetectorType[]{DetectorType.FTCAL,DetectorType.FTHODO,DetectorType.FTTRK,DetectorType.LTCC,DetectorType.ECAL,DetectorType.FTOF,
@@ -109,7 +120,6 @@ public final void initDecoder(){
109120
"/daq/tt/rf","/daq/tt/bmt","/daq/tt/fmt","/daq/tt/rich2","/daq/tt/hel","/daq/tt/band","/daq/tt/rtpc",
110121
"/daq/tt/raster","/daq/tt/atof","/daq/tt/ahdc"
111122
});
112-
translationManager.init(tablesTrans);
113123

114124
// ADC waveform fitter translation table
115125
keysFitter = Arrays.asList(new DetectorType[]{DetectorType.FTCAL,DetectorType.FTHODO,DetectorType.FTTRK,DetectorType.FTOF,DetectorType.LTCC,
@@ -122,17 +132,20 @@ public final void initDecoder(){
122132
"/daq/config/fmt","/daq/fadc/hel","/daq/fadc/rf","/daq/fadc/band","/daq/fadc/raster",
123133
"/daq/config/ahdc"
124134
});
125-
fitterManager.init(tablesFitter);
126135

127136
// Data filter list
128137
keysFilter = Arrays.asList(new DetectorType[]{DetectorType.DC});
129-
130-
scalerManager.init(Arrays.asList(new String[]{"/runcontrol/fcup","/runcontrol/slm","/runcontrol/hwp",
131-
"/runcontrol/helicity","/daq/config/scalers/dsc1"}));
132-
138+
133139
keysMicromega = Arrays.asList(new DetectorType[]{DetectorType.BMT,DetectorType.FMT,DetectorType.FTTRK});
134140

135-
checkTables();
141+
if (initializeManagers) {
142+
translationManager.init(tablesTrans);
143+
fitterManager.init(tablesFitter);
144+
scalerManager.init(Arrays.asList(new String[]{"/runcontrol/fcup","/runcontrol/slm","/runcontrol/hwp",
145+
"/runcontrol/helicity","/daq/config/scalers/dsc1"}));
146+
checkTables();
147+
}
148+
136149
}
137150

138151
public void checkTables() {
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package org.jlab.clas.reco;
2+
3+
import java.util.Set;
4+
import java.util.HashSet;
5+
import java.nio.ByteBuffer;
6+
import java.nio.ByteOrder;
7+
import java.util.concurrent.ArrayBlockingQueue;
8+
import java.util.concurrent.BlockingQueue;
9+
import org.jlab.clara.base.ClaraUtil;
10+
import org.jlab.clara.engine.Engine;
11+
import org.jlab.clara.engine.EngineData;
12+
import org.jlab.clara.engine.EngineDataType;
13+
import org.jlab.clara.engine.EngineStatus;
14+
import org.jlab.detector.decode.CLASDecoder;
15+
import org.jlab.io.evio.EvioDataEvent;
16+
import org.jlab.io.hipo.HipoDataEvent;
17+
import org.jlab.jnp.hipo4.data.SchemaFactory;
18+
import org.json.JSONObject;
19+
20+
/**
21+
*
22+
* @author baltzell
23+
*/
24+
public class DecoderEngine implements Engine {
25+
26+
static final int POOL_SIZE = 64;
27+
static final Set<EngineDataType> ED_TYPES = ClaraUtil.buildDataTypes(
28+
Clas12Types.EVIO,Clas12Types.HIPO,EngineDataType.JSON,EngineDataType.STRING);
29+
30+
SchemaFactory schema;
31+
BlockingQueue<CLASDecoder> pool;
32+
int constantsShared = 64;
33+
34+
public DecoderEngine() {
35+
schema = new SchemaFactory();
36+
schema.initFromDirectory(System.getenv("CLAS12DIR") + "/etc/bankdefs/hipo4");
37+
}
38+
39+
@Override
40+
public Set<EngineDataType> getInputDataTypes() { return ED_TYPES; }
41+
@Override
42+
public Set<EngineDataType> getOutputDataTypes() { return ED_TYPES; }
43+
@Override
44+
public EngineData executeGroup(Set<EngineData> set) { return null; }
45+
@Override
46+
public Set<String> getStates() { return new HashSet<>(); }
47+
@Override
48+
public String getDescription() { return "decoder engine"; }
49+
@Override
50+
public String getVersion() { return "1.0"; }
51+
@Override
52+
public String getAuthor() { return "baltzell"; }
53+
@Override
54+
public void reset() {}
55+
@Override
56+
public void destroy() {}
57+
58+
@Override
59+
public EngineData configure(EngineData ed) {
60+
JSONObject json = new JSONObject(ed.getData());
61+
pool = new ArrayBlockingQueue<>(POOL_SIZE);
62+
CLASDecoder d0 = null;
63+
for (int i=0; i<POOL_SIZE; i++) {
64+
CLASDecoder d;
65+
if (i % constantsShared == 0) {
66+
d0 = new CLASDecoder();
67+
if (json.has("variation")) d0.setVariation(json.getString("variation"));
68+
if (json.has("timestamp")) d0.setVariation(json.getString("timestamp"));
69+
d = d0;
70+
}
71+
else {
72+
d = new CLASDecoder(d0);
73+
}
74+
pool.add(d);
75+
}
76+
return ed;
77+
}
78+
79+
@Override
80+
public EngineData execute(EngineData input) {
81+
82+
EngineData output = input;
83+
84+
// if it's EVIO, decode it, otherwise just pass it along
85+
if (input.getMimeType().equals("binary/data-evio")) {
86+
EvioDataEvent evio;
87+
try {
88+
ByteBuffer bb = (ByteBuffer) input.getData();
89+
//evio = new EvioDataEvent(bb.array(), bb.order());
90+
evio = new EvioDataEvent(bb.array(), ByteOrder.LITTLE_ENDIAN);
91+
} catch (Exception e) {
92+
String msg = String.format("Error reading input event%n%n%s", ClaraUtil.reportException(e));
93+
output.setStatus(EngineStatus.ERROR);
94+
output.setDescription(msg);
95+
return output;
96+
}
97+
HipoDataEvent hipo;
98+
try {
99+
CLASDecoder d = pool.take();
100+
hipo = new HipoDataEvent(d.getDecodedEvent(evio),schema);
101+
pool.put(d);
102+
output.setData("binary/data-hipo", hipo.getHipoEvent());
103+
} catch (Exception e) {
104+
String msg = String.format("Error processing input event%n%n%s", ClaraUtil.reportException(e));
105+
output.setStatus(EngineStatus.ERROR);
106+
output.setDescription(msg);
107+
return output;
108+
}
109+
}
110+
111+
return output;
112+
}
113+
}

‎etc/services/rgd-clarode.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ configuration:
1919
outputBankPrefix: "HB"
2020
io-services:
2121
reader:
22-
class: org.jlab.io.clara.DecoderReader
23-
name: DecoderReader
22+
class: org.jlab.io.clara.EvioToEvioReader
23+
name: EvioReader
2424
writer:
2525
class: org.jlab.io.clara.DecoderWriter
2626
name: DecoderWriter
2727
services:
28+
- class: org.jlab.clas.reco.DecoderEngine
29+
name: DECO
2830
- class: org.jlab.service.ai.DCDenoiseEngine
2931
name: DCDN
3032
- class: org.jlab.clas.swimtools.MagFieldsEngine

0 commit comments

Comments
 (0)