| ID |
Date |
Author |
Subject |
|
286
|
Wed Apr 28 17:39:35 2021 |
TD | Preparations for setup of aida13-aida16 |
Updated /etc/dhcpd.conf
cd /etc
cp dhcpd.conf dhcpd.conf.BAK-280421
Add MAC addresses https://elog.ph.ed.ac.uk/DESPEC/285
aida13 xilinx_lltemac 81c00000.ethernet: MAC address is now d8:80:39:42: d:15
aida14 xilinx_lltemac 81c00000.ethernet: MAC address is now d8:80:39:42: d: b
aida15 xilinx_lltemac 81c00000.ethernet: MAC address is now d8:80:39:41:ee:10
aida16 xilinx_lltemac 81c00000.ethernet: MAC address is now d8:80:39:41:f6:ed
New version of dhcpd.conf (and backup) - attachments 1 & 2
Cannot find dhcpd (and therefore re-read config and restart) in System - Administration -> Services
- has the dhcpd service been re-named or amalgamated with another service?
Updated /MIDAS/config/TclHttpd/aidas-gsi@8015/startup.tcl
New version of startup.tcl (and backup) - attachments 3 & 4
Created rfs filesytems for aida13-aida24
[root@aidas-gsi]# cd /home/Embedded/XilinxLinux/ppc_4xx/rfs
[root@aidas-gsi rfs]# cd aida01
[root@aidas-gsi rfs]# tar cvf ../rfs.tar .
Create directories aida13-aida24, uid=10101, gid=npg, mode 777
[root@aidas-gsi rfs]# cd aida13; tar xvf ../rfs.tar .
:
:
etc
[root@aidas-gsi rfs]# ls -l
total 1491880
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida01
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida02
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida03
drwxrwxrwx. 19 10101 npg 4096 Apr 24 03:51 aida04
drwxrwxrwx. 19 10101 npg 4096 Apr 24 03:51 aida05
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida06
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida07
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida08
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:35 aida09
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida10
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:34 aida11
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida12
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida13
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida14
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida15
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida16
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida17
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida18
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida19
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida20
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida21
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida22
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida23
drwxrwxrwx. 19 10101 npg 4096 Apr 28 16:32 aida24
-rw-r--r--. 1 root root 1527582720 Apr 28 18:26 rfs.tar
Once dhcpd has been restarted it should be possible to power cycle all FEE64s, start system
and update firmware of newly installed FEE64s. |
| Attachment 1: dhcpd.conf
|
#
# DHCP Server Configuration file.
# see /usr/share/doc/dhcp*/dhcpd.conf.sample
#
# Date of last update Jan 12 2015
#
authoritative;
ddns-update-style none; ddns-updates off;
# 2 days
#default-lease-time 172800;
# 4 days
default-lease-time 345600;
# 8 days
max-lease-time 691200;
option domain-search code 119 = string;
option domain-name "dl.ac.uk";
option domain-name-servers 193.62.115.16, 148.79.80.78;
option netbios-name-servers 148.79.160.89;
option netbios-node-type 8;
option nis-domain "nuclear.physics";
option nis-servers 193.62.115.77;
subnet 192.168.11.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option broadcast-address 192.168.11.255;
pool {
range 192.168.11.118 192.168.11.199;
}
}
group {
use-host-decl-names true;
default-lease-time 3600;
max-lease-time 14400;
server-name "192.168.11.99";
next-server 192.168.11.99;
host nnrpi1 {
hardware ethernet b8:27:eb:bb:46:7b;
fixed-address 192.168.11.251;
}
host nnrpi2 {
hardware ethernet b8:27:eb:40:53:e8;
fixed-address 192.168.11.117;
}
host aida01 {
hardware ethernet d8:80:39:41:ba:8a;
fixed-address 192.168.11.1;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida01";
}
host aida02 {
hardware ethernet d8:80:39:41:ba:22;
fixed-address 192.168.11.2;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida02";
}
host aida03 {
hardware ethernet d8:80:39:41:d8:21;
fixed-address 192.168.11.3;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida03";
}
host aida04 {
hardware ethernet d8:80:39:41:f6:b7;
fixed-address 192.168.11.4;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida04";
}
host aida05 {
hardware ethernet d8:80:39:41:d7:cc;
fixed-address 192.168.11.5;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida05";
}
host aida06 {
hardware ethernet d8:80:39:41:ee:72;
fixed-address 192.168.11.6;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida06";
}
host aida07 {
hardware ethernet d8:80:39:41:b4:0c;
fixed-address 192.168.11.7;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida07";
}
host aida08 {
hardware ethernet d8:80:39:41:ba:2b;
fixed-address 192.168.11.8;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida08";
}
host aida09 {
hardware ethernet d8:80:39:41:f6:ee;
fixed-address 192.168.11.9;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida09";
}
host aida10 {
hardware ethernet d8:80:39:41:ba:89;
fixed-address 192.168.11.10;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida10";
}
host aida11 {
hardware ethernet d8:80:39:41:f6:5a;
fixed-address 192.168.11.11;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida11";
}
host aida12 {
hardware ethernet d8:80:39:41:cf:ac;
fixed-address 192.168.11.12;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida12";
}
host aida13 {
hardware ethernet d8:80:39:42:d:15;
fixed-address 192.168.11.13;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida13";
}
host aida14 {
hardware ethernet d8:80:39:42:d:b;
fixed-address 192.168.11.14;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida14";
}
host aida15 {
hardware ethernet d8:80:39:41:ee:10;
fixed-address 192.168.11.15
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida15";
}
host aida16 {
hardware ethernet d8:80:39:41:f6:ed;
fixed-address 192.168.11.16;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida16";
}
}
subnet 192.168.12.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option broadcast-address 192.168.12.255;
pool {
range 192.168.12.100 192.168.12.199;
}
}
group {
use-host-decl-names true;
default-lease-time 3600;
max-lease-time 14400;
server-name "192.168.12.99";
next-server 192.168.12.99;
host aida21 {
hardware ethernet d8:80:39:41:d8:2a;
fixed-address 192.168.12.1;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida21";
}
host aida22 {
hardware ethernet 00:04:a3:2a:d0:26;
fixed-address 192.168.12.2;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida22";
}
host aida23 {
hardware ethernet 00:04:a3:2b:09:ce;
fixed-address 192.168.12.3;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida23";
}
host aida24 {
hardware ethernet 00:04:a3:2b:09:e8;
fixed-address 192.168.12.4;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida24";
}
}
|
| Attachment 2: dhcpd.conf.BAK-280421
|
#
# DHCP Server Configuration file.
# see /usr/share/doc/dhcp*/dhcpd.conf.sample
#
# Date of last update Jan 12 2015
#
authoritative;
ddns-update-style none; ddns-updates off;
# 2 days
#default-lease-time 172800;
# 4 days
default-lease-time 345600;
# 8 days
max-lease-time 691200;
option domain-search code 119 = string;
option domain-name "dl.ac.uk";
option domain-name-servers 193.62.115.16, 148.79.80.78;
option netbios-name-servers 148.79.160.89;
option netbios-node-type 8;
option nis-domain "nuclear.physics";
option nis-servers 193.62.115.77;
subnet 192.168.11.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option broadcast-address 192.168.11.255;
pool {
range 192.168.11.118 192.168.11.199;
}
}
group {
use-host-decl-names true;
default-lease-time 3600;
max-lease-time 14400;
server-name "192.168.11.99";
next-server 192.168.11.99;
host nnrpi1 {
hardware ethernet b8:27:eb:bb:46:7b;
fixed-address 192.168.11.251;
}
host nnrpi2 {
hardware ethernet b8:27:eb:40:53:e8;
fixed-address 192.168.11.117;
}
host aida01 {
hardware ethernet d8:80:39:41:ba:8a;
fixed-address 192.168.11.1;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida01";
}
host aida02 {
hardware ethernet d8:80:39:41:ba:22;
fixed-address 192.168.11.2;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida02";
}
host aida03 {
hardware ethernet d8:80:39:41:d8:21;
fixed-address 192.168.11.3;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida03";
}
host aida04 {
hardware ethernet d8:80:39:41:f6:b7;
fixed-address 192.168.11.4;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida04";
}
host aida05 {
hardware ethernet d8:80:39:41:d7:cc;
fixed-address 192.168.11.5;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida05";
}
host aida06 {
hardware ethernet d8:80:39:41:ee:72;
fixed-address 192.168.11.6;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida06";
}
host aida07 {
hardware ethernet d8:80:39:41:b4:0c;
fixed-address 192.168.11.7;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida07";
}
host aida08 {
hardware ethernet d8:80:39:41:ba:2b;
fixed-address 192.168.11.8;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida08";
}
host aida09 {
hardware ethernet d8:80:39:41:f6:ee;
fixed-address 192.168.11.9;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida09";
}
host aida10 {
hardware ethernet d8:80:39:41:ba:89;
fixed-address 192.168.11.10;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida10";
}
host aida11 {
hardware ethernet d8:80:39:41:f6:5a;
fixed-address 192.168.11.11;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida11";
}
host aida12 {
hardware ethernet d8:80:39:41:cf:ac;
fixed-address 192.168.11.12;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida12";
}
}
subnet 192.168.12.0 netmask 255.255.255.0 {
option subnet-mask 255.255.255.0;
option broadcast-address 192.168.12.255;
pool {
range 192.168.12.100 192.168.12.199;
}
}
group {
use-host-decl-names true;
default-lease-time 3600;
max-lease-time 14400;
server-name "192.168.12.99";
next-server 192.168.12.99;
host aida21 {
hardware ethernet d8:80:39:41:d8:2a;
fixed-address 192.168.12.1;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida21";
}
host aida22 {
hardware ethernet 00:04:a3:2a:d0:26;
fixed-address 192.168.12.2;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida22";
}
host aida23 {
hardware ethernet 00:04:a3:2b:09:ce;
fixed-address 192.168.12.3;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida23";
}
host aida24 {
hardware ethernet 00:04:a3:2b:09:e8;
fixed-address 192.168.12.4;
option root-path "/home/Embedded/XilinxLinux/ppc_4xx/rfs/aida24";
}
}
|
| Attachment 3: startup.tcl
|
Setting BGColor "BlanchedAlmond"
Setting TextColor "red"
Setting ProjectName AIDA
Setting ProjectImage AIDA.gif
Setting ProjectURL "http://npg.dl.ac.uk/NPG/Projects/Fair.html"
Setting Option1Name STFC
Setting Option1URL "stfc.ac.uk"
Setting Option1Image stfc.jpg
# global MessageLoggerServerHost
# set MessageLoggerServerHost nndhcp077.dl.ac.uk
# package require MessageLoggerClient
# Setting Option3Name "Elog Server"
# Setting Option3URL "http://nndhcp077.dl.ac.uk:8080/DataAcq/"
# Install required services
if {![namespace exists ::AIDA]} {namespace eval ::AIDA {puts "created namespace ::AIDA"}}
# set ::AIDA::TestMode 1
global SpectrumServerHost
set SpectrumServerHost aidas
package require XAIDAAccessClient ;# for Client end - ie AIDA server workstation sets MIDAS_XAIDAAccessClient_Exists (SOAP)
package require XAD9252AccessClient ;# for Client end - ie AIDA server workstation sets MIDAS_XAD9252AccessClient_Exists (SOAP)
namespace eval DataAcquisition {
variable PROJECT AIDA
variable ACQSERVERS
set ACQSERVERS [list aida01 aida02 aida03 aida04 aida05 aida06 aida07 aida08 aida09 aida10 aida11 aida12 aida13 aida14 aida15 aida16]
# set ACQSERVERS [list aida01 aida02 aida03 aida04]
# set ACQSERVERS [list aida01 aida02 aida03 aida04 aida05 aida06 aida07 aida08 aida09 aida10 aida11 aida12]
# set ACQSERVERS [list aida01 aida02 aida03 aida04]
variable ACQSERVER [first $ACQSERVERS]
variable DataTransferStreams; set DataTransferStreams 2
# set ::DataAcquisition::EnableMerger 1
# set ::DataAcquisition::EnableTape 1
variable RDOGo 0
}
namespace eval AIDA {
variable ModuleMenu; set ModuleMenu $::DataAcquisition::ACQSERVERS
variable TSMaster; set TSMaster aida01
variable FEE64 [first $ModuleMenu]
variable ConfigBase; set ConfigBase [file join $env(MIDASBASE) config TclHttpd]
}
Setting DataBaseRoot "/MIDAS/DB"
package require DataBaseAccessServer
package require NetVarsServer 1.1
package require SigTaskServer
# package require MemSasServer
# Install services for Run Control
namespace eval $::DataAcquisition::PROJECT {
variable DBRootName [file join EXPERIMENTS AIDA]
variable DBNodeBase 2012
}
package require AIDARunControl
# package require TapeServer
# package require MergerControl
# package require MERGERunControl
Setting Option7Name "Project Home Page"
Setting Option7URL "http://npg.dl.ac.uk/NPG/Projects/Fair.html"
Setting Option8Name "Server Software Home Page"
Setting Option8URL "http://npg.dl.ac.uk/MIDAS/download/WebServices.html"
|
| Attachment 4: startup.tcl.BAK-280421
|
Setting BGColor "BlanchedAlmond"
Setting TextColor "red"
Setting ProjectName AIDA
Setting ProjectImage AIDA.gif
Setting ProjectURL "http://npg.dl.ac.uk/NPG/Projects/Fair.html"
Setting Option1Name STFC
Setting Option1URL "stfc.ac.uk"
Setting Option1Image stfc.jpg
# global MessageLoggerServerHost
# set MessageLoggerServerHost nndhcp077.dl.ac.uk
# package require MessageLoggerClient
# Setting Option3Name "Elog Server"
# Setting Option3URL "http://nndhcp077.dl.ac.uk:8080/DataAcq/"
# Install required services
if {![namespace exists ::AIDA]} {namespace eval ::AIDA {puts "created namespace ::AIDA"}}
# set ::AIDA::TestMode 1
global SpectrumServerHost
set SpectrumServerHost aidas
package require XAIDAAccessClient ;# for Client end - ie AIDA server workstation sets MIDAS_XAIDAAccessClient_Exists (SOAP)
package require XAD9252AccessClient ;# for Client end - ie AIDA server workstation sets MIDAS_XAD9252AccessClient_Exists (SOAP)
namespace eval DataAcquisition {
variable PROJECT AIDA
variable ACQSERVERS
# set ACQSERVERS [list aida01 aida02 aida03 aida04 aida05 aida06 aida07 aida08 aida09 aida10 aida11 aida12 aida13 aida14 aida15 aida16]
# set ACQSERVERS [list aida01 aida02 aida03 aida04]
set ACQSERVERS [list aida01 aida02 aida03 aida04 aida05 aida06 aida07 aida08 aida09 aida10 aida11 aida12]
# set ACQSERVERS [list aida01 aida02 aida03 aida04]
variable ACQSERVER [first $ACQSERVERS]
variable DataTransferStreams; set DataTransferStreams 2
# set ::DataAcquisition::EnableMerger 1
# set ::DataAcquisition::EnableTape 1
variable RDOGo 0
}
namespace eval AIDA {
variable ModuleMenu; set ModuleMenu $::DataAcquisition::ACQSERVERS
variable TSMaster; set TSMaster aida01
variable FEE64 [first $ModuleMenu]
variable ConfigBase; set ConfigBase [file join $env(MIDASBASE) config TclHttpd]
}
Setting DataBaseRoot "/MIDAS/DB"
package require DataBaseAccessServer
package require NetVarsServer 1.1
package require SigTaskServer
# package require MemSasServer
# Install services for Run Control
namespace eval $::DataAcquisition::PROJECT {
variable DBRootName [file join EXPERIMENTS AIDA]
variable DBNodeBase 2012
}
package require AIDARunControl
# package require TapeServer
# package require MergerControl
# package require MERGERunControl
Setting Option7Name "Project Home Page"
Setting Option7URL "http://npg.dl.ac.uk/NPG/Projects/Fair.html"
Setting Option8Name "Server Software Home Page"
Setting Option8URL "http://npg.dl.ac.uk/MIDAS/download/WebServices.html"
|
|
288
|
Fri Apr 30 08:38:12 2021 |
TD | Friday 30 April |
09.40 All system wide checks OK *except*
FEE64 module aida07 failed
Calibration test result: Passed 15, Failed 1
If any modules fail calibration , check the clock status and open the FADC Align and Control browser page to rerun calibration for that module
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1025 Last changed Thu Apr 29 17:50:06 CEST 2021
FEE : aida02 => Options file size is 1014 Last changed Thu Apr 29 14:43:46 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1014 Last changed Thu Apr 29 14:43:53 CEST 2021
FEE : aida05 => Options file size is 1014 Last changed Thu Apr 29 14:43:55 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1014 Last changed Thu Apr 29 14:44:05 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
09.44 DSSSD bias & leakage currents Ok - attachment 1
Merger time seq errors - attachment 2
no merger errors since restart yesterday
FEE64 temps OK - attachment 3
good event stats - attachment 4
Merger/Tape Server server consoles - attachment 5
Grafana DSSSD bias |& leakage currents for previous 7 days - attachment 6
DAQ continues OK - no data storage |
| Attachment 1: 310.png
|
|
| Attachment 2: 311.png
|
|
| Attachment 3: Screenshot_2021-04-30_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 4: Screenshot_2021-04-30_Statistics_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-04-30_NewMerger_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-04-30_AIDA_Alerting_-_Grafana.png
|
|
|
289
|
Sat May 1 08:45:19 2021 |
TD | AIDA 24cm x 8cm DSSSD + triPlast assembly for S496 |
Photos provided by Nic Hubbard and Helena Albers from the assembly of 2x triPlast
plus 2x MSL type BB18(DS)-1000 24cm x 8cm detector assembly - attachments 1-3.
Spacers installed between MSL type BB18(DS)-1000 24cm x 8cm PCBs - attachment 4.
AIDA support stand in 24cm x 8cm DSSSD mode ready for installation of snout assembly - attachments 5 & 6
AIDA 24cm x 8cm DSSSD plus 2x TriPlast installed in S4 - view downstream towards upstream window
of AIDA detctor assembly - attachment 7
AIDA 24cm x 8cm DSSSD plus 2x TriPlast snout assembly installed at S4 - attachment 8
Side view of AIDA + triPLast + FATIMA + DEGAS - attachment 9 |
| Attachment 1: MicrosoftTeams-image.png
|
|
| Attachment 2: IMG_7649.JPG
|
|
| Attachment 3: IMG_7650.JPG
|
|
| Attachment 4: MicrosoftTeams-image_(1).png
|
.png.png "MicrosoftTeams-image_(1).png")
|
| Attachment 5: image-7becce0a-c5bb-431b-8af9-597e2b8d3829.jpg
|
|
| Attachment 6: image-3ba4cdee-a8f4-48e4-b087-03ddf07504e5.jpg
|
|
| Attachment 7: image-beb082d4-a965-453f-8804-394ce65a2f70.jpg
|
|
| Attachment 8: IMG_7697.JPG
|
|
| Attachment 9: IMG_7708.JPG
|
|
| Attachment 10: IMG_7682.JPG
|
|
| Attachment 11: IMG_7681.JPG
|
|
| Attachment 12: FATIMA_Triple_AIDA.png
|
|
|
296
|
Wed May 5 16:57:48 2021 |
TD | Tuesday 5 May |
17.55 Grafana DSSSD bias & leakage current - attachment 1
Note abrupt decrease of DSSSD# 2 leakage current c. 1.5uA at 17.37
DSSSD bias & leakage currents - attachment 2 |
| Attachment 1: Screenshot_2021-05-05_AIDA_Alerting_-_Grafana(1).png
|
.png.png "Screenshot_2021-05-05_AIDA_Alerting_-_Grafana(1).png")
|
| Attachment 2: 401.png
|
|
|
301
|
Sat May 8 09:05:04 2021 |
TD | Saturday 8 May |
09.54 FEE64 temperatures OK - attachment 1
ADC data item stats OK - attachment 2
p+n FEE64s 11x <100k, 1x <200k
Rate spectra - attachments 3 & 4
before and after ASIC check load
1.8.L spectra - attachments 5 & 6
1.8.W spectra 20us FSR - attachments 7 & 8
Disc, WR-28-47, pause, resume info data items - attachments 9-12
note large number pause/resume for aida13 - legacy numbers since last restart/zero?
attachment 16 for pause stats since stats zero at 10.02 - looks OK
Grafana DSSSD bias & leakage currents for most recent 48h OK - atatchment 13
DSSSD bias & leakage currents - attachment 14
Merger stats/merger/tape server messages - attachment 15
10.02 All histograms and stats zero'd
10.06 All system wide checks OK *except*
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Thu May 06 15:29:51 CEST 2021
FEE : aida02 => Options file size is 1014 Last changed Thu Apr 29 14:43:46 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1014 Last changed Thu Apr 29 14:43:53 CEST 2021
FEE : aida05 => Options file size is 1014 Last changed Thu Apr 29 14:43:55 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Wed May 05 12:15:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
10.25 Pulser peak width aida01 97 ch FWHM, aida02 350 ch FWHM
peak widths are increasing cf. earlier this week - see https://elog.ph.ed.ac.uk/DESPEC/291 , https://elog.ph.ed.ac.uk/DESPEC/298 |
| Attachment 1: Screenshot_2021-05-08_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 2: Screenshot_2021-05-08_Statistics_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 4: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 5: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(2).png")
|
| Attachment 6: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(3).png")
|
| Attachment 7: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(4).png")
|
| Attachment 8: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(5).png")
|
| Attachment 9: Screenshot_2021-05-08_Statistics_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(1).png")
|
| Attachment 10: Screenshot_2021-05-08_Statistics_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(2).png")
|
| Attachment 11: Screenshot_2021-05-08_Statistics_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(3).png")
|
| Attachment 12: Screenshot_2021-05-08_Statistics_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(4).png")
|
| Attachment 13: Screenshot_2021-05-08_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 14: 500.png
|
|
| Attachment 15: 501.png
|
|
| Attachment 16: Screenshot_2021-05-08_Statistics_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(5).png")
|
| Attachment 17: Screenshot_2021-05-08_Statistics_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(5).png")
|
| Attachment 18: Screenshot_2021-05-08_Statistics_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(6).png")
|
| Attachment 19: Screenshot_2021-05-08_Temperature_and_status_scan_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-08_Temperature_and_status_scan_aidas-gsi(1).png")
|
| Attachment 20: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(6).png")
|
| Attachment 21: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(7).png")
|
| Attachment 22: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(8).png")
|
| Attachment 23: Screenshot_2021-05-08_Statistics_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(7).png")
|
| Attachment 24: Screenshot_2021-05-08_Statistics_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(8).png")
|
| Attachment 25: Screenshot_2021-05-08_Statistics_aidas-gsi(9).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(9).png")
|
| Attachment 26: Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(9).png
|
.png.png "Screenshot_2021-05-08_Spectrum_Browser_aidas-gsi(9).png")
|
| Attachment 27: Screenshot_2021-05-08_Statistics_aidas-gsi(10).png
|
.png.png "Screenshot_2021-05-08_Statistics_aidas-gsi(10).png")
|
|
302
|
Sun May 9 19:29:23 2021 |
TD | Sunday 9 May |
|
| Attachment 1: Screenshot_2021-05-09_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 2: Screenshot_2021-05-09_Statistics_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-09_Statistics_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-09_Statistics_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-09_Statistics_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-09_Statistics_aidas-gsi(2).png")
|
| Attachment 5: Screenshot_2021-05-09_Statistics_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-09_Statistics_aidas-gsi(3).png")
|
| Attachment 6: Screenshot_2021-05-09_System_wide_Checks_aidas-gsi.png
|
|
| Attachment 7: Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 8: Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 9: Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(2).png")
|
| Attachment 10: Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(3).png")
|
| Attachment 11: Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-09_Spectrum_Browser_aidas-gsi(4).png")
|
| Attachment 12: Screenshot_2021-05-09_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 13: 503.png
|
|
| Attachment 14: 504.png
|
|
|
303
|
Mon May 10 08:49:37 2021 |
TD | Monday 10 May |
09.51 All system wide checks OK *except*
Base Current Difference
aida05 fault 0x8441 : 0x8442 : 1
aida06 fault 0xc5c5 : 0xc5c6 : 1
aida07 fault 0x6f07 : 0x6f08 : 1
aida08 fault 0x4814 : 0x4815 : 1
White Rabbit error counter test result: Passed 12, Failed 4
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 1 4 6 3 5 5 2 3 2 3 6 : 36004
aida02 : 4 1 3 6 3 4 3 3 2 3 6 : 36040
aida03 : 12 8 5 2 3 3 3 4 2 3 6 : 36416
aida04 : 1 2 5 4 1 4 3 4 2 3 6 : 36388
aida05 : 22 8 4 2 4 5 1 4 1 4 6 : 37272
aida06 : 5 4 0 6 3 6 3 3 2 3 6 : 36276
aida07 : 38 16 7 7 1 6 1 3 3 3 6 : 37032
aida08 : 16 4 2 3 1 4 1 3 3 3 6 : 36384
aida09 : 2 3 7 6 5 5 2 3 2 3 6 : 36112
aida10 : 42 15 4 6 2 6 2 4 2 3 6 : 36768
aida11 : 1 5 13 7 3 3 3 3 2 3 6 : 36124
aida12 : 23 9 3 4 7 3 2 3 2 3 6 : 35988
aida13 : 2 6 7 3 4 4 3 2 2 3 6 : 35592
aida14 : 17 11 3 4 3 5 2 3 2 3 6 : 35980
aida15 : 4 7 5 2 1 6 2 3 2 3 6 : 35864
aida16 : 4 1 3 3 4 2 3 4 2 3 6 : 36264
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 09 15:10:52 CEST 2021
FEE : aida02 => Options file size is 1014 Last changed Thu Apr 29 14:43:46 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1014 Last changed Thu Apr 29 14:43:53 CEST 2021
FEE : aida05 => Options file size is 1014 Last changed Thu Apr 29 14:43:55 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Wed May 05 12:15:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
09.52 All histograms, stats and merger stats zero'd
FEE64 temps OK - attachment 1
ADC data, disc/WR 28-27/pause/correlation stats OK - attachments 2-6
Rate spectra - attachment 7
ASIC check load
Rate spectra - attachment 8
1.8.W spectra 20us FSR - attachments 9 & 10
Grafana - most recent 7 days DSSSD bias and leakage current - attachment 11
Merger stats/tape server/merger - attachment 12
DSSSD bias and leakage current OK - attachment 13
10.08 1.8.W spectra 20us, 200us, 2ms & 20ms FSR - attachments 14-21
we are not missing extrinsic noise features at longer time ranges cf. 20us FSR
12.18 Cannot connect to port 8015, MIDAS terminal says "Killed"
Restart MIDAS 8015
Collect FPGA baseline to avoid TCL error
FEEs unaffected and running as before
13.25 FEEs power cycled to see if any change in weird noise/rates situation
No change
ASICs synchronised and back to normal
14.34 Start a pulser walkthrough in May21/R10
Attenuation 5x, Rep Rate 25 Hz
Amplitudes 10..9...1
14.45 Back to 2 Hz @ 2V Pulser, walkthrough complete. No Storage again.
PULSER SETTINGS
---------------------------
Pulse is ON
Positive Tail Pulse
Trigger Source is Internal Clock
Trigger Threshold is 3.5
Amplitude : 2.0 Volts
Rep Rate : 2.0 hZ
Delay : 250.0 ns
Fall Time : 1 ms
Attenuation : 1
Display is : Volts
Equivalent keV is : 200.0 (NB. actually it is 12 MeV)
Ramp Start at 0.01 Volts
Ramp Stop at 9.99 Volts
Ramp Start at 1.0 keV
Ramp Stop at 999.0 keV
Ramp Time is 60 seconds
# Ramp Cycles is 1
OK |
| Attachment 1: Screenshot_2021-05-10_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 2: Screenshot_2021-05-10_Statistics_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-10_Statistics_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-10_Statistics_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-10_Statistics_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-10_Statistics_aidas-gsi(2).png")
|
| Attachment 5: Screenshot_2021-05-10_Statistics_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-10_Statistics_aidas-gsi(3).png")
|
| Attachment 6: Screenshot_2021-05-10_Statistics_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-10_Statistics_aidas-gsi(4).png")
|
| Attachment 7: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 8: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 9: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(2).png")
|
| Attachment 10: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(3).png")
|
| Attachment 11: Screenshot_2021-05-10_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 12: 510.png
|
|
| Attachment 13: 511.png
|
|
| Attachment 14: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(4).png")
|
| Attachment 15: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(11).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(11).png")
|
| Attachment 16: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(5).png")
|
| Attachment 17: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(10).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(10).png")
|
| Attachment 18: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(6).png")
|
| Attachment 19: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(9).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(9).png")
|
| Attachment 20: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(7).png")
|
| Attachment 21: Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-10_Spectrum_Browser_aidas-gsi(8).png")
|
|
318
|
Fri May 14 17:30:56 2021 |
TD | Analysis of S496 files R5_18, R5_32 & R5_45 |
S496/R5_18 - attachment 1
LEC fast discriminators - *all* channels disabled
Slow comparator thresholds - p+n FEE64s 0xa *except* aida09 0xf
n+n FEE64s 0x20
Sampling ADCs OFF
S496/R5_32 - attachment 2
LEC fast discriminators - *all* channels disabled
Slow comparator thresholds - p+n FEE64s 0xa *except* aida09 & aida15 0xf
n+n FEE64s 0x20
Sampling ADCs ON, trigger threshold 15000
S496/R5_45 - attachment 3
LEC fast discriminators - *all* channels disabled
Slow comparator thresholds - p+n FEE64s 0xc
n+n FEE64s 0x20
Sampling ADCs ON, trigger threshold 15000
* N.B. Ignore 'elapsed idle time(s)' metric which does not work with WR timestamps
|
| Attachment 1: R5_18
|
*** TDR format 3.3.0 analyser - TD - January 2019
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 260127700 ( 2662102.7 Hz)
Other data format: 1792300 ( 18342.1 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 1843 ( 18.9 Hz)
RESUME: 1843 ( 18.9 Hz)
SYNC100: 32365 ( 331.2 Hz)
WR48-63: 32365 ( 331.2 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 1723884 ( 17641.9 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 65165 ( 666.9 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 97.715 s
FEE elapsed dead time(s) elapsed idle time(s)
1 36.653 41.693
2 44.272 48.364
3 46.942 50.228
4 0.000 81.057
5 27.042 29.539
6 41.734 45.821
7 43.168 46.980
8 54.980 58.850
9 4.477 17.487
10 57.700 61.776
11 43.911 47.867
12 48.996 52.920
13 21.829 26.853
14 51.720 56.077
15 53.092 56.077
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 16755449 4921 0 0 122 122 2151 2151 0 375 0 4078
1 15729031 4184 0 0 103 103 1989 1989 0 0 0 7687
2 17390463 502955 0 0 136 135 2294 2294 0 498096 0 5652
3 14745088 476937 0 0 118 118 1907 1907 0 472887 0 8350
4 2857057 295831 0 0 0 0 413 413 0 295005 0 2971
5 19341027 128907 0 0 112 112 2426 2426 0 123831 0 5913
6 17649809 199902 0 0 136 136 2168 2168 0 195294 0 3189
7 17833639 143068 0 0 135 135 2201 2201 0 138396 0 5626
8 17514114 4531 0 0 139 138 2127 2127 0 0 0 6021
9 14922420 3686 0 0 23 23 1820 1820 0 0 0 1039
10 16909910 4387 0 0 140 141 2053 2053 0 0 0 3196
11 18661933 4896 0 0 151 151 2297 2297 0 0 0 1392
12 17768788 4718 0 0 148 148 2211 2211 0 0 0 5274
13 16872590 4258 0 0 96 96 2033 2033 0 0 0 605
14 16747047 4335 0 0 134 135 2033 2033 0 0 0 3273
15 18429335 4784 0 0 150 150 2242 2242 0 0 0 899
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time:66231.297s ( 0.483 blocks/s, 0.030 Mb/s)
|
| Attachment 2: R5_32
|
*** TDR format 3.3.0 analyser - TD - January 2019
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 259165945 ( 2447522.6 Hz)
Other data format: 2754055 ( 26008.9 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 1068 ( 10.1 Hz)
RESUME: 1070 ( 10.1 Hz)
SYNC100: 32395 ( 305.9 Hz)
WR48-63: 32395 ( 305.9 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 2687127 ( 25376.8 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 109002 ( 1029.4 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 105.889 s
FEE elapsed dead time(s) elapsed idle time(s)
1 2.772 22.665
2 28.336 32.781
3 9.177 18.264
4 0.000 94.253
5 5.882 26.886
6 8.897 16.463
7 18.512 26.355
8 29.893 33.327
9 0.057 79.535
10 45.010 46.522
11 27.988 27.766
12 32.050 37.161
13 1.981 23.343
14 0.002 96.477
15 36.327 39.608
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 17728100 5332 0 0 32 32 2340 2340 0 588 0 6827
1 15060064 3752 0 0 18 18 1858 1858 0 0 0 14360
2 18674317 723732 0 0 99 99 2382 2382 0 718770 0 8046
3 19118104 904863 0 0 39 39 2463 2463 0 899859 0 15890
4 2260998 321658 0 0 0 0 326 326 0 321006 0 3138
5 14506811 189277 0 0 25 25 1756 1756 0 185715 0 9208
6 21798062 340177 0 0 43 43 2689 2689 0 334713 0 5553
7 18561150 231264 0 0 70 70 2324 2324 0 226476 0 9337
8 22060606 5804 0 0 136 136 2766 2766 0 0 0 10232
9 5139884 1246 0 0 5 5 618 618 0 0 0 1276
10 24215101 6255 0 0 190 191 2937 2937 0 0 0 4375
11 22485598 5956 0 0 148 148 2830 2830 0 0 0 2034
12 19527530 4883 0 0 108 109 2333 2333 0 0 0 8243
13 14237341 3538 0 0 9 9 1760 1760 0 0 0 984
14 2260329 600 0 0 1 1 299 299 0 0 0 8235
15 21531950 5718 0 0 145 145 2714 2714 0 0 0 1264
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time:65849.211s ( 0.486 blocks/s, 0.030 Mb/s)
|
| Attachment 3: R5_45
|
*** TDR format 3.3.0 analyser - TD - January 2019
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 257173968 ( 1605341.2 Hz)
Other data format: 4746032 ( 29625.9 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 149 ( 0.9 Hz)
RESUME: 149 ( 0.9 Hz)
SYNC100: 32597 ( 203.5 Hz)
WR48-63: 32597 ( 203.5 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 4680540 ( 29217.0 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 198582 ( 1239.6 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 160.199 s
FEE elapsed dead time(s) elapsed idle time(s)
1 1.826 60.560
2 1.857 72.569
3 3.101 18.747
4 0.000 149.092
5 0.725 60.979
6 0.276 85.413
7 1.668 27.289
8 2.719 31.235
9 0.000 157.899
10 6.867 10.073
11 2.634 42.733
12 0.375 69.093
13 0.000 148.656
14 0.255 79.878
15 3.499 32.266
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 8318308 3120 0 0 3 3 1077 1077 0 960 0 11152
1 17482671 4536 0 0 10 10 2258 2258 0 0 0 22118
2 12591028 1472109 0 0 7 7 1790 1790 0 1468515 0 17237
3 26467090 1465014 0 0 17 17 3574 3574 0 1457832 0 25917
4 2005748 477188 0 0 0 0 331 331 0 476526 0 4838
5 16244903 298514 0 0 5 5 2012 2012 0 294480 0 14514
6 11816116 567974 0 0 7 7 1554 1554 0 564852 0 9683
7 24437155 423701 0 0 8 8 3155 3155 0 417375 0 17498
8 22741388 5724 0 0 10 10 2852 2852 0 0 0 21729
9 1027419 274 0 0 0 0 137 137 0 0 0 1781
10 39497864 9636 0 0 47 47 4771 4771 0 0 0 12114
11 19835466 4892 0 0 8 8 2438 2438 0 0 0 4107
12 15299334 3708 0 0 8 8 1846 1846 0 0 0 19448
13 3576125 840 0 0 0 0 420 420 0 0 0 1437
14 13120071 3182 0 0 7 7 1584 1584 0 0 0 12115
15 22713282 5620 0 0 12 12 2798 2798 0 0 0 2894
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time:63108.355s ( 0.507 blocks/s, 0.032 Mb/s)
|
|
343
|
Fri May 21 23:19:23 2021 |
TD | Saturday 22 May 00:00-08:00 |
00.16 413Gb remaining /TapeData filesystem
estimate 413/2 x 177s = 10.15h to filesystem full - OK
00.18 All histograms, statistics and merger statistics zero'd
00.57 All system wide checks OK *except*
Base Current Difference
aida05 fault 0x500 : 0x572 : 114
White Rabbit error counter test result: Passed 15, Failed 1
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 19 9 3 4 1 5 2 2 3 3 6 : 36356
aida02 : 4 8 7 2 4 3 3 3 2 3 6 : 35968
aida03 : 12 9 6 2 5 3 2 3 3 3 6 : 36824
aida04 : 11 5 9 2 2 2 2 4 3 3 6 : 37028
aida05 : 24 9 2 1 2 3 2 3 4 3 6 : 37608
aida06 : 25 6 3 4 2 4 3 3 3 3 6 : 37060
aida07 : 3 3 3 1 3 3 2 2 2 2 7 : 37044
aida08 : 20 4 2 3 1 3 2 2 3 3 6 : 36016
aida09 : 15 8 2 5 0 3 3 2 3 3 6 : 36284
aida10 : 16 6 4 1 2 3 3 2 2 4 6 : 37328
aida11 : 20 2 2 1 1 3 2 2 3 3 6 : 35936
aida12 : 1 2 1 2 3 4 1 2 4 3 6 : 36900
aida13 : 25 10 4 1 0 3 2 3 3 3 6 : 36500
aida14 : 20 6 9 2 2 5 1 2 2 4 6 : 37200
aida15 : 31 9 5 2 3 3 1 3 3 3 6 : 36500
aida16 : 24 4 2 3 1 4 2 3 3 3 6 : 36672
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Mon May 17 11:43:04 CEST 2021
FEE : aida02 => Options file size is 1014 Last changed Thu Apr 29 14:43:46 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Wed May 05 12:15:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
01.12 Lost activity monitor - attachment 1
1.8.H spectra - attachments 2 & 3
1.8.L spectra - attachments 4-6
aida01 pulser peak width 120 ch FWHM
Rate specttra - attachments 7 & 8
01.06 Lost activity monitor - attachment 9
ucesb - attachment 10
Grafana - DSSSD bias & leakage currents past 7 days - attachment 11
statistics - adc data, disc data, WR 28-47, pause, correlation scaler, good wave events - attachments 12-17
FEE64 temperatures OK - attachment 18
DSSSD bias & leakage currents OK - attachment 19
Merger/Tape Server/Merger stats - attachment 20
no merger warning/error messages since last restart
01.26 Analysis file S496/R14_1798 - attachment 21
aida08 5.5% deadtime
01.17 Disk space remaining /TapeData filesystem
[npg@aidas-gsi S496]$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_aidasgsi-lv_root
50G 18G 30G 37% /
tmpfs 7.9G 706M 7.2G 9% /dev/shm
/dev/sda2 477M 42M 410M 10% /boot
/dev/sda1 200M 264K 200M 1% /boot/efi
/dev/mapper/vg_aidasgsi-lv_home
401G 90G 292G 24% /home
/dev/mapper/vg_aidas2-lv_home
7.2T 6.4T 395G 95% /media/1e121361-83d3-4825-b6ae-8700b07e0ca7
/dev/sdd1 7.2T 5.6T 1.3T 82% /media/SecondDrive
/dev/sdc1 7.2T 6.6T 259G 97% /media/ThirdDrive
estimate 395/2 x 177s = 9.7h to filesystem full - OK
01.30 despec shift reports no data reported by ucesb
aida stats & tape server ok - no data being forwarded to MBS by data relay
01.39 despec shift restart FATIMA DAQ - data forwarding to MBS by data relay resumes - AIDA file S496/R14_1811
04.35 System wide checks all OK *except*
Base Current Difference
aida05 fault 0x500 : 0x573 : 115
White Rabbit error counter test result: Passed 15, Failed 1
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
[npg@aidas-gsi S496]$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_aidasgsi-lv_root
50G 18G 30G 37% /
tmpfs 7.9G 661M 7.2G 9% /dev/shm
/dev/sda2 477M 42M 410M 10% /boot
/dev/sda1 200M 264K 200M 1% /boot/efi
/dev/mapper/vg_aidasgsi-lv_home
401G 90G 292G 24% /home
/dev/mapper/vg_aidas2-lv_home
7.2T 6.5T 337G 96% /media/1e121361-83d3-4825-b6ae-8700b07e0ca7
/dev/sdd1 7.2T 5.6T 1.3T 82% /media/SecondDrive
/dev/sdc1 7.2T 6.6T 259G 97% /media/ThirdDrive
estimate 337/2 x 177s = 8.3h to filesystem full - OK
FEE64 temperatures OK - attachment 22
ADC data item statistics OK - attachment 24
DSSSD bias & leakage current OK - attachment 26
analysis of file S496/R14_1920 - attachment 28
aida04 deadtime 5.0%
07.10 Ssystem wide checks all OK *except*
Base Current Difference
aida05 fault 0x500 : 0x573 : 115
White Rabbit error counter test result: Passed 15, Failed 1
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
[npg@aidas-gsi S496]$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_aidasgsi-lv_root
50G 18G 30G 37% /
tmpfs 7.9G 662M 7.2G 9% /dev/shm
/dev/sda2 477M 42M 410M 10% /boot
/dev/sda1 200M 264K 200M 1% /boot/efi
/dev/mapper/vg_aidasgsi-lv_home
401G 90G 292G 24% /home
/dev/mapper/vg_aidas2-lv_home
7.2T 6.5T 287G 96% /media/1e121361-83d3-4825-b6ae-8700b07e0ca7
/dev/sdd1 7.2T 5.6T 1.3T 82% /media/SecondDrive
/dev/sdc1 7.2T 6.6T 259G 97% /media/ThirdDrive
estimate 287/2 x 178s = 7.1h to filesystem full - OK
FEE64 temperatures OK - attachment 23
ADC data item statistics OK - attachment 25
DSSSD bias & leakage current OK - attachment 27
analysis of file S496/R14_1920 - attachment 29
aida04 deadtime 4.4%
07.35 Found MBS data monitor kill'd
Restarted MBS data relay and data monitor OK
08.00 beam off - end of S496
08.28 Tape Server -> no storage mode
DAQ continues
09.58 All slow comparators -> 0x64
BNC PB-5 OFF
file S496/R15
alpha background
Tape server 200kb/s, merger 260k data items/s |
| Attachment 1: Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(1).png")
|
| Attachment 2: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(7).png")
|
| Attachment 3: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(6).png")
|
| Attachment 4: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(5).png")
|
| Attachment 5: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(3).png")
|
| Attachment 6: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(2).png")
|
| Attachment 7: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 8: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 9: Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 10: Screenshot_2021-05-22_ucesb.png
|
|
| Attachment 11: Screenshot_2021-05-22_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 12: Screenshot_2021-05-22_Statistics_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(5).png")
|
| Attachment 13: Screenshot_2021-05-22_Statistics_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(4).png")
|
| Attachment 14: Screenshot_2021-05-22_Statistics_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(3).png")
|
| Attachment 15: Screenshot_2021-05-22_Statistics_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(2).png")
|
| Attachment 16: Screenshot_2021-05-22_Statistics_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(1).png")
|
| Attachment 17: Screenshot_2021-05-22_Statistics_aidas-gsi.png
|
|
| Attachment 18: Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 19: 800.png
|
|
| Attachment 20: 801.png
|
|
| Attachment 21: R14_1798
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 254662088 ( 1433466.5 Hz)
Other data format: 7257912 ( 40854.0 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 256 ( 1.4 Hz)
RESUME: 257 ( 1.4 Hz)
SYNC100: 32661 ( 183.8 Hz)
WR48-63: 32661 ( 183.8 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 7192077 ( 40483.5 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 365795 ( 2059.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 177.655 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.112 0.000
1 5.898 0.000
2 0.060 0.000
3 7.669 0.000
4 0.073 7.902
5 7.648 0.000
6 0.356 0.000
7 9.852 0.000
8 1.513 0.000
9 0.000 15.513
10 2.639 0.000
11 0.903 0.000
12 0.687 0.000
13 0.000 6.517
14 0.073 0.000
15 0.212 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 5664289 2546 0 0 5 5 737 737 0 1062 0 19821
1 24002581 6376 0 0 39 39 3149 3149 0 0 0 40581
2 12140787 1878978 0 0 3 3 1758 1758 0 1875456 0 32794
3 29094115 1803103 0 0 54 54 3917 3917 0 1795161 0 46229
4 3034023 1008057 0 0 6 6 480 480 0 1007085 0 8526
5 29014047 561174 0 0 54 54 3636 3636 0 553794 0 24714
6 11191428 1180757 0 0 3 3 1528 1528 0 1177695 0 17521
7 34103710 790567 0 0 51 52 4320 4320 0 781824 0 29424
8 17182670 4348 0 0 9 9 2165 2165 0 0 0 41538
9 2132311 520 0 0 0 0 260 260 0 0 0 3304
10 25432181 6308 0 0 12 12 3142 3142 0 0 0 25046
11 16063496 3832 0 0 8 8 1908 1908 0 0 0 7738
12 11072472 2756 0 0 4 4 1374 1374 0 0 0 35884
13 2789016 664 0 0 0 0 332 332 0 0 0 2683
14 11165568 2934 0 0 4 4 1463 1463 0 0 0 24254
15 20579394 4992 0 0 4 4 2492 2492 0 0 0 5738
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 41.474s ( 771.565 blocks/s, 48.223 Mb/s)
|
| Attachment 22: Screenshot_2021-05-22_Statistics_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(7).png")
|
| Attachment 23: Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(2).png")
|
| Attachment 24: Screenshot_2021-05-22_Statistics_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(6).png")
|
| Attachment 25: Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(1).png")
|
| Attachment 26: 802.png
|
|
| Attachment 27: 804.png
|
|
| Attachment 28: R14_1867
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 254778017 ( 1461534.6 Hz)
Other data format: 7141983 ( 40970.0 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 228 ( 1.3 Hz)
RESUME: 228 ( 1.3 Hz)
SYNC100: 32649 ( 187.3 Hz)
WR48-63: 32649 ( 187.3 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 7076229 ( 40592.8 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 358653 ( 2057.4 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 174.322 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.023 0.000
1 8.071 0.000
2 0.956 0.000
3 6.628 0.000
4 0.186 12.353
5 5.668 0.000
6 0.255 0.000
7 8.669 0.000
8 1.570 0.000
9 0.000 28.420
10 2.125 0.000
11 0.164 0.000
12 0.103 0.000
13 0.000 11.688
14 0.094 0.000
15 1.340 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 5294176 2413 0 0 1 1 685 685 0 1041 0 18715
1 22878067 6136 0 0 44 44 3024 3024 0 0 0 38998
2 11959613 1840531 0 0 7 7 1681 1681 0 1837155 0 31909
3 28914032 1781740 0 0 51 51 3845 3845 0 1773948 0 44969
4 2970776 981532 0 0 7 7 454 454 0 980610 0 8409
5 29112496 551278 0 0 40 40 3688 3688 0 543822 0 24330
6 11438278 1149690 0 0 4 4 1643 1643 0 1146396 0 17361
7 34470364 802023 0 0 41 41 4342 4342 0 793257 0 30254
8 17110979 4312 0 0 6 6 2150 2150 0 0 0 40990
9 2129568 524 0 0 0 0 262 262 0 0 0 3068
10 25946722 6422 0 0 9 9 3202 3202 0 0 0 25071
11 16268948 3960 0 0 3 3 1977 1977 0 0 0 7119
12 11320269 2746 0 0 3 3 1370 1370 0 0 0 35684
13 2799924 716 0 0 0 0 358 358 0 0 0 2469
14 11389376 2772 0 0 2 2 1384 1384 0 0 0 23994
15 20774429 5188 0 0 10 10 2584 2584 0 0 0 5313
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 43.000s ( 744.186 blocks/s, 46.512 Mb/s)
|
| Attachment 29: R14_1920
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 254781729 ( 1428365.9 Hz)
Other data format: 7138271 ( 40018.8 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 226 ( 1.3 Hz)
RESUME: 227 ( 1.3 Hz)
SYNC100: 32665 ( 183.1 Hz)
WR48-63: 32665 ( 183.1 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 7072488 ( 39650.0 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 352885 ( 1978.4 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 178.373 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.017 2.836
1 6.610 0.000
2 0.573 0.000
3 7.826 0.000
4 0.024 14.831
5 7.215 0.000
6 0.482 0.000
7 6.876 0.000
8 0.394 0.000
9 0.000 17.832
10 1.157 0.000
11 1.395 0.000
12 0.037 0.000
13 0.004 9.016
14 0.101 0.000
15 1.578 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 5306415 2397 0 0 1 1 665 665 0 1065 0 18998
1 22746335 6008 0 0 36 36 2968 2968 0 0 0 37812
2 11931383 1875609 0 0 5 5 1756 1756 0 1872087 0 31542
3 28584441 1795513 0 0 57 57 3773 3773 0 1787853 0 43417
4 2954259 971818 0 0 3 3 494 494 0 970824 0 8522
5 28608195 543853 0 0 51 52 3567 3567 0 536616 0 24072
6 11355091 1133078 0 0 6 6 1573 1573 0 1129920 0 16979
7 34582740 783003 0 0 34 34 4406 4406 0 774123 0 29465
8 17425265 4332 0 0 7 7 2159 2159 0 0 0 40811
9 2130016 550 0 0 0 0 275 275 0 0 0 3176
10 26903024 6616 0 0 4 4 3304 3304 0 0 0 24219
11 16400506 4036 0 0 5 5 2013 2013 0 0 0 7163
12 11085724 2934 0 0 1 1 1466 1466 0 0 0 35361
13 2664218 650 0 0 1 1 324 324 0 0 0 2574
14 11252748 2738 0 0 4 4 1365 1365 0 0 0 23421
15 20851369 5136 0 0 11 11 2557 2557 0 0 0 5353
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 43.672s ( 732.737 blocks/s, 45.796 Mb/s)
|
|
344
|
Sat May 22 13:40:56 2021 |
TD | Saturday 22 May |
14.37 DAQ continues file S496/R15_14
Grafana - DSSSD bias & leakage current - attachment 1
1.8.L spectra - attachments 2 & 3
Stat spectra - attachments 4 & 5
FEE64 temperatures OK - attachment 6
stats - ADC data items, pause items OK - attachments 7 & 8
DSSSD bias & leakage current OK - attachment 9
Merger/Tape Server/Merger stats - attachment 10
analysis of file S496/R14_1920 - attachment 11
all FEE64 deadtime 0% as expected
17.30 aida01, 3, 5, 9 & 11 have hot HEC channels
ASIC check fixes
All histogram, statistics & merger statistics zero'd
DAQ continues
Note gamma-ray source tests have started with FATIMA and DEGAS
20.20 Analysis file R15_19 - attachment 12
[npg@aidas-gsi S496]$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_aidasgsi-lv_root
50G 18G 30G 37% /
tmpfs 7.9G 665M 7.2G 9% /dev/shm
/dev/sda2 477M 42M 410M 10% /boot
/dev/sda1 200M 264K 200M 1% /boot/efi
/dev/mapper/vg_aidasgsi-lv_home
401G 90G 292G 24% /home
/dev/mapper/vg_aidas2-lv_home
7.2T 6.3T 530G 93% /media/1e121361-83d3-4825-b6ae-8700b07e0ca7
/dev/sdd1 7.2T 5.6T 1.3T 82% /media/SecondDrive
/dev/sdc1 7.2T 6.6T 259G 97% /media/ThirdDrive
estimate 90/2 x 3168s = 40h to filesystem full - OK
20.22 All system wide checks OK *except*
Base Current Difference
aida01 fault 0xf82c : 0xf82f : 3
aida02 fault 0xdda6 : 0xdda9 : 3
aida03 fault 0xb13d : 0xb140 : 3
aida04 fault 0xf4cd : 0xf4d0 : 3
aida05 fault 0x500 : 0x575 : 117
aida06 fault 0xc6e7 : 0xc6e9 : 2
aida07 fault 0x4920 : 0x4922 : 2
aida08 fault 0xbd9a : 0xbd9c : 2
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 8 4 6 6 1 4 3 3 4 3 6 : 38048
aida02 : 11 8 10 3 4 4 3 4 3 3 6 : 37740
aida03 : 2 8 3 3 2 2 1 4 2 4 6 : 37720
aida04 : 26 15 17 4 4 2 1 4 2 4 6 : 38256
aida05 : 15 6 4 1 1 2 1 3 3 4 6 : 38156
aida06 : 7 7 16 6 3 4 2 2 3 4 6 : 38612
aida07 : 11 5 12 3 3 3 2 3 1 3 7 : 38836
aida08 : 13 6 7 2 1 2 2 2 3 4 6 : 37972
aida09 : 23 3 8 5 0 5 3 3 4 3 6 : 38164
aida10 : 12 5 7 1 2 2 4 3 2 4 6 : 37992
aida11 : 19 2 7 1 2 2 1 2 3 4 6 : 37740
aida12 : 21 5 3 3 3 3 1 2 3 4 6 : 37964
aida13 : 18 7 7 1 2 4 2 3 2 4 6 : 37776
aida14 : 8 3 13 7 2 4 2 3 2 4 6 : 37992
aida15 : 11 5 14 1 3 4 2 3 2 4 6 : 37908
aida16 : 17 6 6 2 2 3 1 2 3 4 6 : 37908
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Mon May 17 11:43:04 CEST 2021
FEE : aida02 => Options file size is 1014 Last changed Thu Apr 29 14:43:46 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Wed May 05 12:15:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
19.30 Lost activity monitors - attachments 13-15
20.54 1.8.W spectra 20us FSR attachments 16 & 23
1.8.W spectra 200us FSR attachments 17 & 22
1.8.W spectra 2ms FSR attachments 18 & 21
1.8.W spectra 20ms FSR attachments 19 & 20 |
| Attachment 1: Screenshot_2021-05-22_AIDA_Alerting_-_Grafana(1).png
|
.png.png "Screenshot_2021-05-22_AIDA_Alerting_-_Grafana(1).png")
|
| Attachment 2: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(11).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(11).png")
|
| Attachment 3: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(10).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(10).png")
|
| Attachment 4: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(9).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(9).png")
|
| Attachment 5: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(8).png")
|
| Attachment 6: Screenshot_2021-05-22_Statistics_aidas-gsi(9).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(9).png")
|
| Attachment 7: Screenshot_2021-05-22_Statistics_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-22_Statistics_aidas-gsi(8).png")
|
| Attachment 8: Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-22_Temperature_and_status_scan_aidas-gsi(3).png")
|
| Attachment 9: 805.png
|
|
| Attachment 10: 806.png
|
|
| Attachment 11: R15_13
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 102550190 ( 10129.0 Hz)
Other data format: 159369822 ( 15741.1 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 69714 ( 6.9 Hz)
WR48-63: 69714 ( 6.9 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 159230394 ( 15727.3 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 10124.450 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 0.000
2 0.000 67.951
3 0.000 0.000
4 0.000 0.000
5 0.000 9971.491
6 0.000 0.000
7 0.000 10069.837
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 213 0 0 0 0 0 0 0 0 0 0 0
1 1159 2 0 0 0 0 1 1 0 0 0 0
2 279 79660137 0 0 0 0 22470 22470 0 79615197 0 0
3 101219504 79708933 0 0 0 0 46868 46868 0 79615197 0 0
4 246 0 0 0 0 0 0 0 0 0 0 0
5 588847 366 0 0 0 0 183 183 0 0 0 0
6 280 0 0 0 0 0 0 0 0 0 0 0
7 737827 382 0 0 0 0 191 191 0 0 0 0
8 262 0 0 0 0 0 0 0 0 0 0 0
9 221 0 0 0 0 0 0 0 0 0 0 0
10 242 2 0 0 0 0 1 1 0 0 0 0
11 210 0 0 0 0 0 0 0 0 0 0 0
12 273 0 0 0 0 0 0 0 0 0 0 0
13 193 0 0 0 0 0 0 0 0 0 0 0
14 250 0 0 0 0 0 0 0 0 0 0 0
15 184 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 45.816s ( 698.440 blocks/s, 43.652 Mb/s)
|
| Attachment 12: R15_19
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 55871232 ( 17633.8 Hz)
Other data format: 206048770 ( 65032.0 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 56 ( 0.0 Hz)
RESUME: 56 ( 0.0 Hz)
SYNC100: 43803 ( 13.8 Hz)
WR48-63: 43803 ( 13.8 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 205961052 ( 65004.3 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 3168.420 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2865.072
2 0.000 0.000
3 0.419 0.000
4 0.000 0.000
5 0.000 2166.184
6 0.000 0.000
7 0.000 1949.169
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 55 0 0 0 0 0 0 0 0 0 0 0
1 133932 56 0 0 0 0 28 28 0 0 0 0
2 84 103025372 0 0 0 0 18982 18982 0 102987408 0 0
3 42389344 103017790 0 0 56 56 22017 22017 0 102973644 0 0
4 56 0 0 0 0 0 0 0 0 0 0 0
5 6040914 2574 0 0 0 0 1287 1287 0 0 0 0
6 103 0 0 0 0 0 0 0 0 0 0 0
7 7306081 2976 0 0 0 0 1488 1488 0 0 0 0
8 78 0 0 0 0 0 0 0 0 0 0 0
9 86 0 0 0 0 0 0 0 0 0 0 0
10 68 0 0 0 0 0 0 0 0 0 0 0
11 75 0 0 0 0 0 0 0 0 0 0 0
12 116 2 0 0 0 0 1 1 0 0 0 0
13 80 0 0 0 0 0 0 0 0 0 0 0
14 80 0 0 0 0 0 0 0 0 0 0 0
15 80 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 43.406s ( 737.221 blocks/s, 46.076 Mb/s)
|
| Attachment 13: Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(4).png")
|
| Attachment 14: Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(3).png")
|
| Attachment 15: Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-22_Lost_activity_monitor_aidas-gsi(2).png")
|
| Attachment 16: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(19).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(19).png")
|
| Attachment 17: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(18).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(18).png")
|
| Attachment 18: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(17).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(17).png")
|
| Attachment 19: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(16).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(16).png")
|
| Attachment 20: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(15).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(15).png")
|
| Attachment 21: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(14).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(14).png")
|
| Attachment 22: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(13).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(13).png")
|
| Attachment 23: Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(12).png
|
.png.png "Screenshot_2021-05-22_Spectrum_Browser_aidas-gsi(12).png")
|
|
345
|
Sat May 22 23:40:32 2021 |
TD | Sunday 23 May |
00.40 Unable to connect to FEE64s 1-7
DAQ power cycled OK
ASIC settings 2021Apr29-13-16-00
slow comparator 0x64 *all* FEE64s
BNC PB-5 OFF
file S496/R16
alpha background
07.17 Some hot HEC channels
ASIC check fixes issue
System wide checks OK *except*
Base Current Difference
aida06 fault 0x5457 : 0x5457 : 0
aida06 : WR status 0x10
White Rabbit error counter test result: Passed 15, Failed 1
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Base Current Difference
aida12 fault 0x0 : 0x1cd : 461
FPGA Timestamp error counter test result: Passed 15, Failed 1
If any of these counts are reported as in error
The ASIC readout system has detected a timeslip.
That is the timestamp read from the time FIFO is not younger than the last
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 16 9 4 3 1 4 3 2 1 4 7 : 40552
aida02 : 3 2 2 1 2 3 3 2 2 4 7 : 41308
aida03 : 14 5 0 1 1 3 2 4 2 3 7 : 40000
aida04 : 14 4 5 1 3 3 1 4 2 3 7 : 39944
aida05 : 12 4 1 3 2 3 3 2 2 4 7 : 41408
aida06 : 8 3 2 1 2 4 2 3 3 4 7 : 42744
aida07 : 10 4 3 2 1 2 2 3 2 4 7 : 41464
aida08 : 7 5 2 0 3 3 3 2 2 4 7 : 41380
aida09 : 10 5 1 1 1 2 2 3 2 4 7 : 41408
aida10 : 11 6 3 2 1 3 2 2 2 4 7 : 41100
aida11 : 12 8 3 2 2 2 3 2 1 4 7 : 40288
aida12 : 9 9 3 1 2 3 3 2 1 4 7 : 40380
aida13 : 10 4 0 1 1 2 1 3 2 4 7 : 41128
aida14 : 9 4 2 1 1 2 1 3 2 4 7 : 41156
aida15 : 5 4 0 3 3 3 3 3 2 4 7 : 41940
aida16 : 9 6 3 2 0 2 1 3 2 4 7 : 41156
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 00:29:48 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE64 temperatures OK - attachment 1
ADC data item stats - attachment 2
Grafana DSSSD bias & leakage currents most recent 7 days - attachment 3
DSSSD bias & leakage currents OK - attachment 4
11.18 Analysis of file S496/R16_33 - attachment 5
[npg@aidas-gsi S496]$ df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_aidasgsi-lv_root
50G 18G 30G 37% /
tmpfs 7.9G 685M 7.2G 9% /dev/shm
/dev/sda2 477M 42M 410M 10% /boot
/dev/sda1 200M 264K 200M 1% /boot/efi
/dev/mapper/vg_aidasgsi-lv_home
401G 90G 292G 24% /home
/dev/mapper/vg_aidas2-lv_home
7.2T 6.0T 867G 88% /media/1e121361-83d3-4825-b6ae-8700b07e0ca7
/dev/sdd1 7.2T 5.6T 1.3T 82% /media/SecondDrive
/dev/sdc1 7.2T 6.6T 259G 97% /media/ThirdDrive
867/2 x 3279s = 16.5 days to filesystem full - OK
12.11 aida15 zero data/poll stats - re-loaded DAQ program OK
aida04 unable to connect - multiple merger starts ineffective - aida04 rebooted and DAQ re-started OK
DAQ continues file S496/R18 |
| Attachment 1: Screenshot_2021-05-23_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-23_Statistics_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-23_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 4: 810.png
|
|
| Attachment 5: R16_33
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 47385414 ( 14450.2 Hz)
Other data format: 214534594 ( 65422.3 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 61 ( 0.0 Hz)
RESUME: 61 ( 0.0 Hz)
SYNC100: 44215 ( 13.5 Hz)
WR48-63: 44215 ( 13.5 Hz)
FEE64 disc: 1311957 ( 400.1 Hz)
MBS info: 213134085 ( 64995.2 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 3279.230 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 0.000
2 0.386 0.000
3 1.000 0.000
4 0.000 0.000
5 0.000 2849.824
6 0.000 0.000
7 0.000 2608.689
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 71 0 0 0 0 0 0 0 0 0 0 0
1 1455 0 0 0 0 0 0 0 0 0 0 0
2 87 106617422 0 0 35 35 20186 20186 0 106576980 0 0
3 39184250 107913590 0 0 26 26 22238 22238 1311957 106557105 0 0
4 63 0 0 0 0 0 0 0 0 0 0 0
5 3345685 1512 0 0 0 0 756 756 0 0 0 0
6 60 0 0 0 0 0 0 0 0 0 0 0
7 4853274 2070 0 0 0 0 1035 1035 0 0 0 0
8 69 0 0 0 0 0 0 0 0 0 0 0
9 45 0 0 0 0 0 0 0 0 0 0 0
10 70 0 0 0 0 0 0 0 0 0 0 0
11 68 0 0 0 0 0 0 0 0 0 0 0
12 89 0 0 0 0 0 0 0 0 0 0 0
13 57 0 0 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0 0 0 0
15 71 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 44.293s ( 722.462 blocks/s, 45.154 Mb/s)
|
|
346
|
Mon May 24 09:57:07 2021 |
TD | Monday 24 May |
07.30 Hot HEC channels multiple FEE64s
ASIC check fixes
10.56 DAQ continues OK file S496/R18_75
System wide checks OK * except*
FEE64 module aida06 global clocks failed, 6
Clock status test result: Passed 15, Failed 1
Understand status as follows
Status bit 3 : firmware PLL that creates clocks from external clock not locked
Status bit 2 : always logic '1'
Status bit 1 : LMK3200(2) PLL and clock distribution chip not locked to external clock
Status bit 0 : LMK3200(1) PLL and clock distribution chip not locked to external clock
If all these bits are not set then the operation of the firmware is unreliable
Base Current Difference
aida01 fault 0x82b0 : 0x82b1 : 1
aida02 fault 0xfe72 : 0xfe73 : 1
aida03 fault 0x97f4 : 0x97f5 : 1
aida04 fault 0x950 : 0x951 : 1
aida05 fault 0x8e93 : 0x8e94 : 1
aida06 fault 0x545a : 0x545b : 1
aida07 fault 0xc04d : 0xc04e : 1
aida08 fault 0xa0f7 : 0xa0f8 : 1
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 1 2 7 2 1 4 3 2 1 4 7 : 40452
aida02 : 13 3 3 1 3 4 2 2 2 4 7 : 41308
aida03 : 17 6 2 2 1 4 2 3 2 3 7 : 39700
aida04 : 8 6 1 0 1 3 1 4 2 3 7 : 39712
aida05 : 10 3 5 3 4 3 3 2 1 4 7 : 40560
aida06 : 4 4 2 0 2 3 2 3 3 4 7 : 42576
aida07 : 5 3 2 3 1 3 1 3 2 4 7 : 41324
aida08 : 11 5 1 1 2 3 3 2 2 4 7 : 41348
aida09 : 7 4 3 0 2 3 1 3 2 4 7 : 41324
aida10 : 11 4 2 2 2 3 2 4 1 4 7 : 41132
aida11 : 18 5 0 2 2 4 2 2 1 4 7 : 40240
aida12 : 2 3 0 1 2 3 2 2 2 4 7 : 41024
aida13 : 4 5 1 2 2 3 2 4 1 4 7 : 41096
aida14 : 9 3 1 2 1 4 2 2 2 4 7 : 41164
aida15 : 2 3 1 3 2 2 3 3 2 4 7 : 41744
aida16 : 5 5 2 0 3 3 2 4 1 4 7 : 41116
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
Statistics - disc, pause, WR 28-47, correlation scaler - attachments 2-5
1.8.H spectra - attachments 6-8
1.8.L spectra - attachments 9 & 10
Stat spectra - attachments 11 & 12
1.8.W spectra - 20us FSR - attachments 13 & 14
ADC data items - attachment 15
FEE64 temperatures OK - attachment 16
DSSSD bias & leakage currents OK - attachment 17
Merger/Tape Server/Merger statistics - attachment 18
no merger errors reported since previous restart
analysis of file R18_74
max dead time 0.05%
10.17 ASIC check
All histograms, statistics, merger statistics zero'd |
| Attachment 1: Screenshot_2021-05-24_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-24_Statistics_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-24_Statistics_aidas-gsi(4).png")
|
| Attachment 3: Screenshot_2021-05-24_Statistics_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-24_Statistics_aidas-gsi(3).png")
|
| Attachment 4: Screenshot_2021-05-24_Statistics_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-24_Statistics_aidas-gsi(2).png")
|
| Attachment 5: Screenshot_2021-05-24_Statistics_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-24_Statistics_aidas-gsi(1).png")
|
| Attachment 6: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(8).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(8).png")
|
| Attachment 7: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(7).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(7).png")
|
| Attachment 8: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(6).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(6).png")
|
| Attachment 9: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(5).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(5).png")
|
| Attachment 10: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(4).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(4).png")
|
| Attachment 11: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(3).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(3).png")
|
| Attachment 12: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(2).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(2).png")
|
| Attachment 13: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 14: Screenshot_2021-05-24_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 15: Screenshot_2021-05-24_Statistics_aidas-gsi.png
|
|
| Attachment 16: Screenshot_2021-05-24_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 17: 900.png
|
|
| Attachment 18: 901.png
|
|
| Attachment 19: R18_74
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 42982076 ( 14466.2 Hz)
Other data format: 218937925 ( 73686.5 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 59 ( 0.0 Hz)
RESUME: 59 ( 0.0 Hz)
SYNC100: 43069 ( 14.5 Hz)
WR48-63: 43069 ( 14.5 Hz)
FEE64 disc: 1163669 ( 391.6 Hz)
MBS info: 217688000 ( 73265.8 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2971.209 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 0.000
2 0.425 0.000
3 1.488 0.000
4 0.000 0.000
5 0.000 2622.333
6 0.000 0.000
7 0.000 2340.476
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 64 0 0 0 0 0 0 0 0 0 0 0
1 2227 3 0 0 0 0 1 1 1 0 0 0
2 75 108903724 0 0 25 25 19964 19964 0 108863746 0 0
3 35357926 110031040 0 0 34 34 21525 21525 1163668 108824254 0 0
4 89 0 0 0 0 0 0 0 0 0 0 0
5 3145189 1316 0 0 0 0 658 658 0 0 0 0
6 64 0 0 0 0 0 0 0 0 0 0 0
7 4475895 1842 0 0 0 0 921 921 0 0 0 0
8 56 0 0 0 0 0 0 0 0 0 0 0
9 67 0 0 0 0 0 0 0 0 0 0 0
10 115 0 0 0 0 0 0 0 0 0 0 0
11 65 0 0 0 0 0 0 0 0 0 0 0
12 65 0 0 0 0 0 0 0 0 0 0 0
13 77 0 0 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0 0 0 0
15 102 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 40.766s ( 784.975 blocks/s, 49.061 Mb/s)
|
|
347
|
Tue May 25 11:16:54 2021 |
TD | Tuesday 25 May |
12.16 DAQ continues file S496/R18_324
alpha background
Tape server c. 2.5Mb/s
ASIC check
Tape server c. 300kb/s - presumably hot HEC channels
12.20 All system wide checks OK *except*
Base Current Difference
aida01 fault 0x82b0 : 0x82b1 : 1
aida02 fault 0xfe72 : 0xfe73 : 1
aida03 fault 0x97f4 : 0x97f5 : 1
aida04 fault 0x950 : 0x951 : 1
aida05 fault 0x8e93 : 0x8e94 : 1
aida06 fault 0x545a : 0x545b : 1
aida07 fault 0xc04d : 0xc04e : 1
aida08 fault 0xa0f7 : 0xa0f8 : 1
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 2 6 6 2 1 3 2 2 2 4 6 : 37016
aida02 : 3 6 5 3 3 4 3 2 1 3 7 : 38572
aida03 : 21 10 8 6 3 3 2 3 2 4 6 : 37924
aida04 : 29 15 4 5 3 4 2 3 3 3 6 : 37004
aida05 : 17 5 2 2 2 2 2 2 2 4 6 : 36940
aida06 : 1 6 0 1 2 2 2 3 3 4 6 : 38356
aida07 : 11 6 7 4 3 3 3 3 2 4 6 : 38028
aida08 : 13 8 3 2 2 3 3 2 1 3 7 : 38372
aida09 : 7 9 2 1 3 3 3 3 2 4 6 : 37860
aida10 : 9 8 4 2 0 3 3 3 1 4 6 : 36708
aida11 : 16 6 3 2 2 3 3 3 1 4 6 : 36832
aida12 : 21 10 7 4 2 4 2 2 1 3 7 : 38420
aida13 : 8 3 3 1 3 3 3 3 2 4 6 : 37832
aida14 : 14 3 6 1 0 4 2 2 2 4 6 : 37072
aida15 : 2 4 2 2 1 3 3 4 2 4 6 : 38216
aida16 : 16 5 5 3 0 3 2 2 2 4 6 : 37016
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage cuuent - most recent 7 days - attachment 1
Rate spectra - attachment 2
ADC data item stats - attachment 3
FEE64 temps OK - attachment 4
DSSSD bias & leakage current OK - attachment 5
Merger/Tape Server/Merger stats - attachment 6
no merger errors reported since previous restart
17.30 analysis of file S496/R18_340
max deadtime 0.36% aida04 |
| Attachment 1: Screenshot_2021-05-25_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-25_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-25_Statistics_aidas-gsi.png
|
|
| Attachment 4: Screenshot_2021-05-25_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 5: 910.png
|
|
| Attachment 6: 911.png
|
|
| Attachment 7: R18_340
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 63855540 ( 23748.1 Hz)
Other data format: 198064462 ( 73660.8 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 109 ( 0.0 Hz)
RESUME: 109 ( 0.0 Hz)
SYNC100: 42015 ( 15.6 Hz)
WR48-63: 42015 ( 15.6 Hz)
FEE64 disc: 1291036 ( 480.1 Hz)
MBS info: 196689178 ( 73149.3 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 35 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2688.871 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2304.866
2 0.344 0.000
3 9.625 0.000
4 0.000 0.000
5 0.000 1450.665
6 0.000 0.000
7 0.000 1243.447
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 303 7 0 0 0 0 1 1 5 0 0 5
1 3031185 1240 0 0 0 0 620 620 0 0 0 0
2 536 98548687 0 0 20 20 16872 16872 2 98514901 0 3
3 44821969 99507955 0 0 89 89 21248 21248 1291004 98174277 0 0
4 514 3 0 0 0 0 1 1 1 0 0 1
5 7529304 3056 0 0 0 0 1528 1528 0 0 0 0
6 1314 2 0 0 0 0 0 0 2 0 0 2
7 8467764 3488 0 0 0 0 1744 1744 0 0 0 0
8 226 4 0 0 0 0 0 0 4 0 0 4
9 144 4 0 0 0 0 0 0 4 0 0 4
10 524 1 0 0 0 0 0 0 1 0 0 0
11 432 2 0 0 0 0 0 0 2 0 0 2
12 719 0 0 0 0 0 0 0 0 0 0 0
13 154 7 0 0 0 0 1 1 5 0 0 7
14 0 0 0 0 0 0 0 0 0 0 0 0
15 452 6 0 0 0 0 0 0 6 0 0 7
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 39.492s ( 810.287 blocks/s, 50.643 Mb/s)
|
|
348
|
Wed May 26 13:21:59 2021 |
TD | Wednesday 26 May |
14.22 DAQ continues - file S496/R18_369
alpha background
All system wide checks OK *except*
Base Current Difference
aida01 fault 0x82b0 : 0x82b1 : 1
aida02 fault 0xfe72 : 0xfe73 : 1
aida03 fault 0x97f4 : 0x97f5 : 1
aida04 fault 0x950 : 0x951 : 1
aida05 fault 0x8e93 : 0x8e94 : 1
aida06 fault 0x545a : 0x545b : 1
aida07 fault 0xc04d : 0xc04e : 1
aida08 fault 0xa0f7 : 0xa0f8 : 1
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 12 4 6 1 2 3 2 2 2 4 6 : 37072
aida02 : 12 2 4 1 3 4 3 3 2 4 6 : 37984
aida03 : 12 6 7 6 2 4 3 2 3 3 6 : 36624
aida04 : 36 11 4 6 4 3 2 3 3 3 6 : 36968
aida05 : 2 4 6 3 2 2 2 2 2 4 6 : 36968
aida06 : 8 5 3 2 2 2 2 3 3 4 6 : 38456
aida07 : 10 4 6 4 3 3 3 3 2 4 6 : 37992
aida08 : 21 8 3 2 3 3 3 3 2 4 6 : 37956
aida09 : 17 10 3 2 3 4 3 3 1 4 6 : 37060
aida10 : 3 6 10 2 1 3 3 3 1 4 6 : 36828
aida11 : 12 6 4 2 2 3 3 3 1 4 6 : 36832
aida12 : 2 9 5 1 3 3 2 3 2 4 6 : 37632
aida13 : 4 7 2 1 3 3 3 3 2 4 6 : 37832
aida14 : 15 6 6 1 0 4 2 2 2 4 6 : 37100
aida15 : 5 4 3 2 1 3 3 4 2 4 6 : 38244
aida16 : 17 6 6 2 1 3 2 2 2 4 6 : 37076
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage cuuent - most recent 7 days - attachment 1
Lost activity monitor - attachment 2
1.8.W spectra - attachments 3 & 4
ADC data item stats - attachment 5
FEE64 temps OK - attachment 6
DSSSD bias & leakage current OK - attachment 7
Merger/Tape Server/Merger stats - attachment 8
no merger errors reported since previous restart
14.30 analysis of file S496/R18_340 - attachment 9
max deadtime 0.31% aida04 |
| Attachment 1: Screenshot_2021-05-26_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-26_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-26_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-26_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-26_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-05-26_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-05-26_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 7: 920.png
|
|
| Attachment 8: 921.png
|
|
| Attachment 9: R18_368
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 60865731 ( 22299.7 Hz)
Other data format: 201054280 ( 73661.2 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 152 ( 0.1 Hz)
RESUME: 152 ( 0.1 Hz)
SYNC100: 42166 ( 15.4 Hz)
WR48-63: 42166 ( 15.4 Hz)
FEE64 disc: 1309767 ( 479.9 Hz)
MBS info: 199659877 ( 73150.3 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 107 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2729.447 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2401.079
2 0.453 0.000
3 8.570 0.000
4 0.000 0.000
5 0.000 1507.937
6 0.000 0.000
7 0.000 1499.170
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 1115 29 0 0 0 0 1 1 27 0 0 10
1 2766811 11645 0 0 0 0 574 574 10497 0 0 18
2 2194 100014163 0 0 27 27 17354 17354 0 99979401 0 1
3 44009553 100997602 0 0 125 125 21286 21286 1274310 99680470 0 5
4 902 6 0 0 0 0 0 0 0 6 0 0
5 6872755 14651 0 0 0 0 1451 1451 11749 0 0 10
6 1948 13 0 0 0 0 1 1 11 0 0 9
7 7202790 16106 0 0 0 0 1497 1497 13112 0 0 9
8 531 0 0 0 0 0 0 0 0 0 0 0
9 394 0 0 0 0 0 0 0 0 0 0 3
10 1118 1 0 0 0 0 0 0 1 0 0 0
11 900 26 0 0 0 0 0 0 26 0 0 12
12 1756 13 0 0 0 0 0 0 13 0 0 8
13 1346 11 0 0 0 0 1 1 9 0 0 7
14 0 0 0 0 0 0 0 0 0 0 0 0
15 1618 14 0 0 0 0 1 1 12 0 0 15
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 31.695s ( 1009.613 blocks/s, 63.101 Mb/s)
|
|
349
|
Thu May 27 12:25:53 2021 |
TD | Thursday 27 May |
13.29 DAQ continues - file S496/R18_443
alpha background
All system wide checks OK *except*
FEE64 module aida06 global clocks failed, 6
Clock status test result: Passed 15, Failed 1
Understand status as follows
Status bit 3 : firmware PLL that creates clocks from external clock not locked
Status bit 2 : always logic '1'
Status bit 1 : LMK3200(2) PLL and clock distribution chip not locked to external clock
Status bit 0 : LMK3200(1) PLL and clock distribution chip not locked to external clock
If all these bits are not set then the operation of the firmware is unreliable
Base Current Difference
aida01 fault 0x82b0 : 0x82b2 : 2
aida02 fault 0xfe72 : 0xfe74 : 2
aida03 fault 0x97f4 : 0x97f6 : 2
aida04 fault 0x950 : 0x952 : 2
aida05 fault 0x8e93 : 0x8e95 : 2
aida06 fault 0x545a : 0x545c : 2
aida07 fault 0xc04d : 0xc04f : 2
aida08 fault 0xa0f7 : 0xa0f9 : 2
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 2 7 7 1 3 2 2 2 2 4 6 : 37008
aida02 : 3 1 5 1 3 4 3 3 2 4 6 : 37956
aida03 : 2 6 5 6 2 4 2 3 3 3 6 : 36808
aida04 : 24 12 5 5 4 3 2 3 3 3 6 : 36912
aida05 : 8 1 9 3 2 2 2 2 2 4 6 : 37016
aida06 : 3 6 2 2 2 2 2 3 3 4 6 : 38428
aida07 : 6 17 1 2 0 3 3 2 2 4 6 : 37232
aida08 : 4 2 1 2 3 4 3 3 2 4 6 : 37936
aida09 : 15 7 5 2 3 4 3 3 1 4 6 : 37060
aida10 : 7 6 9 2 0 3 3 3 1 4 6 : 36764
aida11 : 19 10 2 0 2 3 3 3 1 4 6 : 36796
aida12 : 11 6 4 2 2 4 2 3 2 4 6 : 37724
aida13 : 17 9 2 0 0 4 2 2 2 4 6 : 37036
aida14 : 15 4 7 1 1 3 2 2 2 4 6 : 37036
aida15 : 9 6 1 2 1 3 3 4 2 4 6 : 38244
aida16 : 11 11 8 2 0 3 2 2 2 4 6 : 37060
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
Lost activity monitor - attachment 2
1.8.W spectra - attachments 3 & 4
ADC data item stats - attachment 5
FEE64 temps OK - attachment 6
DSSSD bias & leakage current OK - attachment 7
Merger/Tape Server/Merger stats - attachment 8
no merger errors reported since previous restart
14.30 analysis of file S496/R18_442 - attachment 9
max deadtime 0.28% aida04 |
| Attachment 1: Screenshot_2021-05-27_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-27_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-27_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-27_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-27_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-05-27_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-05-27_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 7: 930.png
|
|
| Attachment 8: 931.png
|
|
| Attachment 9: R18_442
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 68693190 ( 21525.3 Hz)
Other data format: 193226813 ( 60548.5 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 114 ( 0.0 Hz)
RESUME: 114 ( 0.0 Hz)
SYNC100: 43889 ( 13.8 Hz)
WR48-63: 43889 ( 13.8 Hz)
FEE64 disc: 1335474 ( 418.5 Hz)
MBS info: 191803333 ( 60102.4 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 3 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 3191.275 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2989.890
2 0.511 643.077
3 8.760 0.000
4 0.000 0.000
5 0.000 1578.752
6 0.000 0.000
7 0.000 1340.150
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 155 0 0 0 0 0 0 0 0 0 0 0
1 527962 216 0 0 0 0 108 108 0 0 0 0
2 533 96091128 0 0 28 28 16832 16832 0 96057408 0 0
3 49735236 97127126 0 0 86 86 22778 22778 1335473 95745925 0 0
4 118 2 0 0 0 0 1 1 0 0 0 0
5 8268444 3874 0 0 0 0 1937 1937 0 0 0 0
6 632 0 0 0 0 0 0 0 0 0 0 0
7 10156762 4466 0 0 0 0 2233 2233 0 0 0 0
8 500 0 0 0 0 0 0 0 0 0 0 0
9 102 0 0 0 0 0 0 0 0 0 0 0
10 1498 0 0 0 0 0 0 0 0 0 0 0
11 105 0 0 0 0 0 0 0 0 0 0 0
12 886 0 0 0 0 0 0 0 0 0 0 0
13 95 1 0 0 0 0 0 0 1 0 0 2
14 0 0 0 0 0 0 0 0 0 0 0 0
15 162 0 0 0 0 0 0 0 0 0 0 1
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 32.027s ( 999.146 blocks/s, 62.447 Mb/s)
|
|
350
|
Fri May 28 10:29:07 2021 |
TD | Friday 28 May |
11.30 DAQ continues - file S496/R18_472
alpha background
All system wide checks OK *except*
FEE64 module aida06 global clocks failed, 6
Clock status test result: Passed 15, Failed 1
Understand status as follows
Status bit 3 : firmware PLL that creates clocks from external clock not locked
Status bit 2 : always logic '1'
Status bit 1 : LMK3200(2) PLL and clock distribution chip not locked to external clock
Status bit 0 : LMK3200(1) PLL and clock distribution chip not locked to external clock
If all these bits are not set then the operation of the firmware is unreliable
Base Current Difference
aida01 fault 0x82b0 : 0x82b2 : 2
aida02 fault 0xfe72 : 0xfe74 : 2
aida03 fault 0x97f4 : 0x97f6 : 2
aida04 fault 0x950 : 0x952 : 2
aida05 fault 0x8e93 : 0x8e95 : 2
aida06 fault 0x545a : 0x545c : 2
aida07 fault 0xc04d : 0xc04f : 2
aida08 fault 0xa0f7 : 0xa0f9 : 2
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 1 4 7 1 3 2 2 2 2 4 6 : 36980
aida02 : 3 1 5 1 2 4 3 3 2 4 6 : 37892
aida03 : 8 8 6 6 3 3 2 3 3 3 6 : 36800
aida04 : 37 12 5 6 3 4 2 3 3 3 6 : 37060
aida05 : 11 5 6 4 2 2 2 2 2 4 6 : 37044
aida06 : 4 3 3 3 2 2 2 3 3 4 6 : 38456
aida07 : 1 12 3 2 0 3 3 2 2 4 6 : 37204
aida08 : 13 8 1 2 3 4 3 3 2 4 6 : 38020
aida09 : 7 7 5 1 1 3 2 2 2 4 6 : 36996
aida10 : 12 7 9 2 1 3 3 3 1 4 6 : 36856
aida11 : 2 9 3 1 2 3 3 3 1 4 6 : 36768
aida12 : 11 4 3 3 2 4 2 3 2 4 6 : 37724
aida13 : 10 8 6 2 2 3 2 2 2 4 6 : 37128
aida14 : 16 7 7 1 0 4 2 2 2 4 6 : 37128
aida15 : 12 6 2 2 1 3 3 4 2 4 6 : 38272
aida16 : 9 4 10 1 2 3 2 2 2 4 6 : 37124
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
Lost activity monitor - attachment 2
1.8.W spectra - attachments 3 & 4
ADC data item stats - attachment 5
FEE64 temps OK - attachment 6
DSSSD bias & leakage current OK - attachment 7
Merger/Tape Server/Merger stats - attachment 8
no merger errors reported since previous restart
14.30 analysis of file S496/R18_471 - attachment 9
max deadtime 0.18% aida04 |
| Attachment 1: Screenshot_2021-05-28_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-28_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-28_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-28_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-28_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-05-28_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-05-28_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 7: 940.png
|
|
| Attachment 8: 941.png
|
|
| Attachment 9: R18_471
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 57407666 ( 20753.5 Hz)
Other data format: 204512336 ( 73933.4 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 116 ( 0.0 Hz)
RESUME: 116 ( 0.0 Hz)
SYNC100: 42303 ( 15.3 Hz)
WR48-63: 42303 ( 15.3 Hz)
FEE64 disc: 1160567 ( 419.6 Hz)
MBS info: 203266931 ( 73483.1 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 109 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2766.172 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2700.364
2 0.332 0.000
3 4.856 0.000
4 0.000 0.000
5 0.000 1685.309
6 0.000 0.000
7 0.000 1567.327
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 181 23 0 0 0 0 0 0 23 0 0 13
1 838470 386 0 0 0 0 166 166 54 0 0 0
2 162 101753181 0 0 19 19 17931 17931 15 101717266 0 5
3 42392398 102752755 0 0 97 97 21331 21331 1160234 101549665 0 12
4 211 21 0 0 0 0 0 0 21 0 0 10
5 6478076 2740 0 0 0 0 1337 1337 66 0 0 0
6 260 17 0 0 0 0 0 0 17 0 0 4
7 7695649 3163 0 0 0 0 1537 1537 89 0 0 28
8 132 14 0 0 0 0 1 1 12 0 0 10
9 368 5 0 0 0 0 0 0 5 0 0 4
10 173 18 0 0 0 0 0 0 18 0 0 5
11 303 5 0 0 0 0 0 0 5 0 0 5
12 186 1 0 0 0 0 0 0 1 0 0 7
13 585 2 0 0 0 0 0 0 2 0 0 2
14 0 0 0 0 0 0 0 0 0 0 0 0
15 512 5 0 0 0 0 0 0 5 0 0 4
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 31.742s ( 1008.122 blocks/s, 63.008 Mb/s)
|
|
351
|
Sat May 29 13:13:45 2021 |
TD | Saturday 29 May |
14.10 DAQ continues - file S496/R18_539
alpha background
increased data rate due to ht HEC channels since c. 06.00 this am - ASIC check fixes problem
All system wide checks OK *except*
Base Current Difference
aida01 fault 0x82b0 : 0x82b7 : 7
aida02 fault 0xfe72 : 0xfe79 : 7
aida03 fault 0x97f4 : 0x97fb : 7
aida04 fault 0x950 : 0x957 : 7
aida05 fault 0x8e93 : 0x8e97 : 4
aida06 fault 0x545a : 0x545e : 4
aida07 fault 0xc04d : 0xc051 : 4
aida08 fault 0xa0f7 : 0xa0fb : 4
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 15 8 7 2 2 2 2 2 2 4 6 : 37036
aida02 : 14 3 5 2 2 4 3 3 2 4 6 : 37984
aida03 : 15 11 8 6 3 3 2 3 3 3 6 : 36884
aida04 : 43 13 5 5 3 4 2 3 3 3 6 : 37060
aida05 : 19 3 5 4 2 2 2 2 2 4 6 : 37044
aida06 : 17 4 1 3 2 3 3 4 2 4 6 : 38356
aida07 : 16 6 2 1 0 2 2 3 2 4 6 : 37296
aida08 : 21 8 1 1 3 4 4 3 2 4 6 : 38276
aida09 : 13 10 2 1 2 3 2 2 2 4 6 : 37060
aida10 : 13 9 4 1 1 3 3 3 1 4 6 : 36764
aida11 : 13 9 2 1 2 3 3 3 1 4 6 : 36796
aida12 : 11 6 4 2 2 4 2 3 2 4 6 : 37724
aida13 : 7 6 6 2 1 3 2 2 2 4 6 : 37036
aida14 : 9 7 7 1 1 3 2 2 2 4 6 : 37036
aida15 : 14 5 2 2 1 3 3 4 2 4 6 : 38272
aida16 : 9 4 10 1 2 3 2 2 2 4 6 : 37124
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
Lost activity monitor - attachment 2
1.8.W spectra - attachments 3 & 4
ADC data item stats - attachment 5
FEE64 temps OK - attachment 6
DSSSD bias & leakage current OK - attachment 7
Merger/Tape Server/Merger stats - attachment 8
no merger errors reported since previous restart
14.24 All histograms, statistics and merger statistics zero'd
16.16 analysis of file S496/R18_541 - attachment 9
max deadtime 0.12% aida04
18.40 DAQ continues - file S496/R18_551
alpha background
increased data rate due to ht HEC channels since c. 18.00 this am - ASIC check fixes problem |
| Attachment 1: Screenshot_2021-05-29_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-29_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-29_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-29_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-29_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-05-29_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-05-29_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 7: 950.png
|
|
| Attachment 8: 951.png
|
|
| Attachment 9: R18_541
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 46792076 ( 18702.9 Hz)
Other data format: 215127929 ( 85987.1 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 89 ( 0.0 Hz)
RESUME: 89 ( 0.0 Hz)
SYNC100: 41318 ( 16.5 Hz)
WR48-63: 41318 ( 16.5 Hz)
FEE64 disc: 1013016 ( 404.9 Hz)
MBS info: 214032099 ( 85549.1 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2501.862 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 1964.282
2 0.467 0.000
3 3.050 0.000
4 0.000 1832.921
5 0.000 844.521
6 0.000 1829.325
7 0.000 869.373
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 499 0 0 0 0 0 0 0 0 0 0 0
1 74891 24 0 0 0 0 12 12 0 0 0 0
2 1032 92085039 0 0 27 27 16251 16251 0 92052483 0 0
3 37271457 93008741 0 0 62 62 19135 19135 1013015 91957332 0 0
4 552 7507421 0 0 0 0 925 925 0 7505571 0 0
5 4051453 7509613 0 0 0 0 2021 2021 0 7505571 0 0
6 1401 7507471 0 0 0 0 950 950 0 7505571 0 0
7 5387708 7509619 0 0 0 0 2024 2024 0 7505571 0 0
8 460 0 0 0 0 0 0 0 0 0 0 0
9 183 0 0 0 0 0 0 0 0 0 0 0
10 537 0 0 0 0 0 0 0 0 0 0 0
11 388 0 0 0 0 0 0 0 0 0 0 0
12 674 0 0 0 0 0 0 0 0 0 0 0
13 244 1 0 0 0 0 0 0 1 0 0 0
14 0 0 0 0 0 0 0 0 0 0 0 0
15 597 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 31.855s ( 1004.537 blocks/s, 62.784 Mb/s)
|
|
352
|
Sun May 30 08:57:53 2021 |
TD | Sunday 30 May |
09.55 DAQ continues - file S496/R18_588
alpha background
increased data rate due to hot HEC channels since c. 06.00 this am - ASIC check fixes problem
All system wide checks OK *except*
Base Current Difference
aida01 fault 0x82b0 : 0x82b7 : 7
aida02 fault 0xfe72 : 0xfe79 : 7
aida03 fault 0x97f4 : 0x97fb : 7
aida04 fault 0x950 : 0x957 : 7
aida05 fault 0x8e93 : 0x8e98 : 5
aida06 fault 0x545a : 0x545f : 5
aida07 fault 0xc04d : 0xc052 : 5
aida08 fault 0xa0f7 : 0xa0fc : 5
White Rabbit error counter test result: Passed 8, Failed 8
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 6 9 5 1 3 2 2 2 2 4 6 : 37008
aida02 : 3 5 5 2 2 4 3 3 2 4 6 : 37956
aida03 : 22 12 7 5 3 3 2 3 3 3 6 : 36872
aida04 : 40 13 5 5 3 4 2 3 3 3 6 : 37048
aida05 : 15 5 5 4 2 2 2 2 2 4 6 : 37044
aida06 : 11 6 2 1 2 3 3 4 2 4 6 : 38300
aida07 : 17 5 4 3 1 3 3 2 2 4 6 : 37324
aida08 : 1 8 1 2 2 4 4 3 2 4 6 : 38164
aida09 : 10 8 6 1 2 4 3 3 1 4 6 : 36968
aida10 : 14 7 5 2 1 3 3 3 1 4 6 : 36800
aida11 : 7 9 3 1 2 3 3 3 1 4 6 : 36788
aida12 : 3 5 2 3 1 4 2 3 2 4 6 : 37620
aida13 : 12 4 4 2 1 3 2 2 2 4 6 : 37008
aida14 : 11 10 5 1 1 3 2 2 2 4 6 : 37036
aida15 : 11 5 1 2 1 3 3 4 2 4 6 : 38244
aida16 : 10 4 8 1 2 3 2 2 2 4 6 : 37096
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Sun May 23 12:04:25 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
Lost activity monitor - attachment 2
1.8.W spectra - attachments 3 & 4
ADC data item stats - attachment 5
FEE64 temps OK - attachment 6
DSSSD bias & leakage current OK - attachment 7
Merger/Tape Server/Merger stats - attachment 8
no merger errors reported since previous restart
14.24 All histograms, statistics and merger statistics zero'd
16.16 analysis of file S496/R18_541 - attachment 9
max deadtime 0.02% aida03
18.40 DAQ continues - file S496/R18_589
alpha background
increased data rate due to hot HEC channels since c. 18.00 this am - ASIC check fixes problem |
| Attachment 1: Screenshot_2021-05-30_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-05-30_Lost_activity_monitor_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-05-30_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-05-30_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 4: Screenshot_2021-05-30_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-05-30_Statistics_aidas-gsi.png
|
|
| Attachment 6: Screenshot_2021-05-30_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 7: 960.png
|
|
| Attachment 8: 961.png
|
|
| Attachment 9: R18_589
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 31985
ADC data format: 58630511 ( 25364.6 Hz)
Other data format: 203166719 ( 87893.5 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 35 ( 0.0 Hz)
RESUME: 35 ( 0.0 Hz)
SYNC100: 40596 ( 17.6 Hz)
WR48-63: 40596 ( 17.6 Hz)
FEE64 disc: 1341777 ( 580.5 Hz)
MBS info: 201743680 ( 87277.8 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2311.511 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 1879.226
2 0.365 0.000
3 0.313 0.000
4 0.000 1697.189
5 0.000 557.330
6 0.000 1686.560
7 0.000 647.171
8 0.000 0.000
9 0.000 0.000
10 0.382 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 157 0 0 0 0 0 0 0 0 0 0 0
1 3314609 1296 0 0 0 0 648 648 0 0 0 0
2 413 87031526 0 0 19 19 14856 14856 0 87001776 0 0
3 42258127 88383449 0 0 15 15 18954 18954 1341775 87003736 0 0
4 153 6936262 0 0 0 0 860 860 0 6934542 0 0
5 6312304 6939162 0 0 0 0 2310 2310 0 6934542 0 0
6 544 6936170 0 0 0 0 814 814 0 6934542 0 0
7 6741331 6938850 0 0 0 0 2154 2154 0 6934542 0 0
8 448 0 0 0 0 0 0 0 0 0 0 0
9 79 0 0 0 0 0 0 0 0 0 0 0
10 1189 2 0 0 1 1 0 0 0 0 0 0
11 111 2 0 0 0 0 0 0 2 0 0 0
12 767 0 0 0 0 0 0 0 0 0 0 0
13 99 0 0 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0 0 0 0
15 180 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 31.504s ( 1015.271 blocks/s, 63.454 Mb/s)
|
|
354
|
Thu Jun 3 10:16:39 2021 |
TD | Thursday 3 June |
11.19 Hot HEC channels multiple FEE64s
ASIC check fixes
System wide checks OK * except*
Base Current Difference
aida01 fault 0x8940 : 0x8941 : 1
aida02 fault 0x9b32 : 0x9b33 : 1
aida03 fault 0xf266 : 0xf267 : 1
aida04 fault 0x579b : 0x579c : 1
White Rabbit error counter test result: Passed 12, Failed 4
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 11 4 2 2 2 2 2 3 2 4 7 : 41516
aida02 : 10 7 2 0 1 2 2 3 2 4 7 : 41408
aida03 : 10 8 2 2 2 4 1 4 2 3 7 : 40008
aida04 : 5 4 4 2 2 3 1 4 2 3 7 : 39860
aida05 : 12 3 2 2 1 2 2 3 2 4 7 : 41448
aida06 : 15 9 2 2 3 2 1 3 2 4 7 : 41380
aida07 : 12 8 1 2 2 3 3 2 2 4 7 : 41408
aida08 : 9 6 3 1 2 3 3 2 2 4 7 : 41380
aida09 : 15 6 1 1 2 2 2 3 2 4 7 : 41500
aida10 : 15 6 3 2 1 3 3 2 1 4 7 : 40348
aida11 : 8 2 1 1 1 3 1 3 2 4 7 : 41248
aida12 : 18 3 2 1 3 3 2 2 2 4 7 : 41184
aida13 : 7 4 1 4 3 4 3 3 1 4 7 : 41100
aida14 : 18 6 1 4 3 4 2 2 1 4 7 : 40392
aida15 : 7 7 1 3 1 2 1 3 2 4 7 : 41220
aida16 : 4 4 2 3 3 3 2 4 1 4 7 : 41200
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Wed Jun 02 16:27:30 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 1
1.8.W spectra - 20us FSR - attachments 2 & 3
ADC data items - attachment 4
FEE64 temperatures OK - attachment 5
DSSSD bias & leakage currents OK - attachment 6
Merger/Tape Server/Merger statistics - attachment 7
no merger errors reported since previous restart
11.43 Fixed merger issue which stopped output from merger - see https://elog.ph.ed.ac.uk/DESPEC/36
13.09 analysis of file S496/R31_16 - attachment 8
max deadtime 0.31% aida04 |
| Attachment 1: Screenshot_2021-06-03_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 2: Screenshot_2021-06-03_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-06-03_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 3: Screenshot_2021-06-03_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 4: Screenshot_2021-06-03_Statistics_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-06-03_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 6: 970.png
|
|
| Attachment 7: 971.png
|
|
| Attachment 8: R31_16
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 50135749 ( 17884.4 Hz)
Other data format: 211784253 ( 75547.7 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 199 ( 0.1 Hz)
RESUME: 199 ( 0.1 Hz)
SYNC100: 42443 ( 15.1 Hz)
WR48-63: 42443 ( 15.1 Hz)
FEE64 disc: 1081490 ( 385.8 Hz)
MBS info: 210617479 ( 75131.4 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 4 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2803.320 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2522.967
2 0.535 0.000
3 8.636 0.000
4 0.000 0.000
5 0.000 2119.634
6 0.000 0.000
7 0.000 2067.036
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 196 2 0 0 0 0 0 0 2 0 0 2
1 193203 74 0 0 0 0 37 37 0 0 0 0
2 471 105497692 0 0 26 26 18605 18605 0 105460430 0 0
3 40159698 106282506 0 0 173 173 21812 21812 1081487 105157049 0 0
4 128 0 0 0 0 0 0 0 0 0 0 0
5 4517663 1914 0 0 0 0 957 957 0 0 0 0
6 641 2 0 0 0 0 1 1 0 0 0 0
7 5260254 2060 0 0 0 0 1030 1030 0 0 0 0
8 463 0 0 0 0 0 0 0 0 0 0 0
9 104 0 0 0 0 0 0 0 0 0 0 0
10 1026 2 0 0 0 0 1 1 0 0 0 0
11 141 1 0 0 0 0 0 0 1 0 0 1
12 786 0 0 0 0 0 0 0 0 0 0 0
13 109 0 0 0 0 0 0 0 0 0 0 0
14 687 0 0 0 0 0 0 0 0 0 0 0
15 179 0 0 0 0 0 0 0 0 0 0 1
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 28.445s ( 1124.966 blocks/s, 70.310 Mb/s)
|
|
355
|
Fri Jun 4 08:40:52 2021 |
TD | Friday 4 June |
09.20 DAQ found stopped
files in /TapeData/S496 indicate increased data rate yesterday evening from c. 18.00 until data stopped at c. midnight
aida15 rebooted at 00.02
regained control of DAQ and issued DAQ STOP by restarting NewMerger *twice*
11.00 DAQ reset and setup
All system wide checks OK *except*
Base Current Difference
aida05 fault 0x4d3c : 0x4d3c : 0
aida05 : WR status 0x10
White Rabbit error counter test result: Passed 15, Failed 1
Understand the status reports as follows:-
Status bit 3 : White Rabbit decoder detected an error in the received data
Status bit 2 : Firmware registered WR error, no reload of Timestamp
Status bit 0 : White Rabbit decoder reports uncertain of Timestamp information from WR
Returned 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Mem(KB) : 4 8 16 32 64 128 256 512 1k 2k 4k
aida01 : 2 4 2 8 5 4 1 2 2 4 7 : 41352
aida02 : 0 3 3 6 6 4 1 2 2 4 7 : 41352
aida03 : 2 4 2 12 1 3 2 2 2 4 7 : 41352
aida04 : 2 2 1 7 4 3 2 2 2 4 7 : 41352
aida05 : 2 3 1 5 3 2 3 2 2 4 7 : 41360
aida06 : 2 4 2 6 4 3 2 2 2 4 7 : 41352
aida07 : 1 5 2 9 4 2 4 3 1 4 7 : 41324
aida08 : 0 3 1 7 4 3 4 3 1 4 7 : 41352
aida09 : 3 3 0 8 2 2 3 2 2 4 7 : 41380
aida10 : 1 2 2 11 3 2 3 3 3 3 7 : 41044
aida11 : 2 3 1 6 6 4 2 3 3 3 7 : 41072
aida12 : 4 2 1 8 3 3 3 3 1 4 7 : 41072
aida13 : 2 4 1 8 4 4 2 3 3 3 7 : 41016
aida14 : 1 4 1 9 4 4 2 3 3 3 7 : 41044
aida15 : 0 5 1 2 2 1 2 4 3 4 7 : 42872
aida16 : 1 3 2 9 4 4 4 4 2 3 7 : 41052
Collecting the file size of each FEE64 Options CONTENTS file to check they are all the same
FEE : aida01 => Options file size is 1026 Last changed Fri Jun 04 09:30:42 CEST 2021
FEE : aida02 => Options file size is 1025 Last changed Sun May 23 00:19:21 CEST 2021
FEE : aida03 => Options file size is 1014 Last changed Thu Apr 29 14:43:50 CEST 2021
FEE : aida04 => Options file size is 1025 Last changed Fri May 14 16:54:56 CEST 2021
FEE : aida05 => Options file size is 1025 Last changed Mon May 17 06:25:41 CEST 2021
FEE : aida06 => Options file size is 1014 Last changed Thu Apr 29 14:43:59 CEST 2021
FEE : aida07 => Options file size is 1014 Last changed Thu Apr 29 14:44:02 CEST 2021
FEE : aida08 => Options file size is 1025 Last changed Sun May 23 00:16:54 CEST 2021
FEE : aida09 => Options file size is 1014 Last changed Thu Apr 29 14:44:08 CEST 2021
FEE : aida10 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida11 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida12 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida13 => Options file size is 1025 Last changed Fri May 07 19:40:34 CEST 2021
FEE : aida14 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida15 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
FEE : aida16 => Options file size is 1014 Last changed Thu Apr 29 14:44:57 CEST 2021
11.03 DAQ resumes file S496/R32
alpha background
slow comparator 0x64
BNC PB-5 pulser OFF
11.09 1.8.W spectra - 20us FSR - attachments 1 & 2
Grafana - DSSSD bias & leakage current - most recent 7 days - attachment 3
ADC data items - attachment 4
FEE64 temperatures OK - attachment 5
DSSSD bias & leakage currents OK - attachment 6
Merger/Tape Server/Merger statistics - attachment 7
no merger errors reported since previous restart
13.07 analysis of file S496/R32_16 - attachment 8
max deadtime 0.18% aida04 |
| Attachment 1: Screenshot_2021-06-04_Spectrum_Browser_aidas-gsi(1).png
|
.png.png "Screenshot_2021-06-04_Spectrum_Browser_aidas-gsi(1).png")
|
| Attachment 2: Screenshot_2021-06-04_Spectrum_Browser_aidas-gsi.png
|
|
| Attachment 3: Screenshot_2021-06-04_AIDA_Alerting_-_Grafana.png
|
|
| Attachment 4: Screenshot_2021-06-04_Statistics_aidas-gsi.png
|
|
| Attachment 5: Screenshot_2021-06-04_Temperature_and_status_scan_aidas-gsi.png
|
|
| Attachment 6: 980.png
|
|
| Attachment 7: 981.png
|
|
| Attachment 8: R32_16
|
*** TDR format 3.3.0 analyser - TD - May 2021
*** ERROR: READ I/O error: 5002
blocks: 32000
ADC data format: 49166189 ( 17465.0 Hz)
Other data format: 212753815 ( 75575.1 Hz)
Sample trace data format: 0 ( 0.0 Hz)
Undefined format: 0 ( 0.0 Hz)
Other data format type: PAUSE: 137 ( 0.0 Hz)
RESUME: 137 ( 0.0 Hz)
SYNC100: 42486 ( 15.1 Hz)
WR48-63: 42486 ( 15.1 Hz)
FEE64 disc: 1037793 ( 368.6 Hz)
MBS info: 211630776 ( 75176.2 Hz)
Other info: 0 ( 0.0 Hz)
ADC data range bit set: 0 ( 0.0 Hz)
Timewarps: ADC: 0 ( 0.0 Hz)
PAUSE: 0 ( 0.0 Hz)
RESUME: 0 ( 0.0 Hz)
SYNC100: 0 ( 0.0 Hz)
WR48-63: 0 ( 0.0 Hz)
FEE64 disc: 0 ( 0.0 Hz)
MBS info: 0 ( 0.0 Hz)
Undefined: 0 ( 0.0 Hz)
Sample trace: 0 ( 0.0 Hz)
*** Timestamp elapsed time: 2815.130 s
FEE elapsed dead time(s) elapsed idle time(s)
0 0.000 0.000
1 0.000 2759.088
2 0.741 0.000
3 4.946 0.000
4 0.000 0.000
5 0.000 2173.993
6 0.000 0.000
7 0.000 2104.432
8 0.000 0.000
9 0.000 0.000
10 0.000 0.000
11 0.000 0.000
12 0.000 0.000
13 0.000 0.000
14 0.000 0.000
15 0.000 0.000
16 0.000 0.000
17 0.000 0.000
18 0.000 0.000
19 0.000 0.000
20 0.000 0.000
21 0.000 0.000
22 0.000 0.000
23 0.000 0.000
24 0.000 0.000
25 0.000 0.000
26 0.000 0.000
27 0.000 0.000
28 0.000 0.000
29 0.000 0.000
30 0.000 0.000
31 0.000 0.000
32 0.000 0.000
*** Statistics
FEE ADC Data Other Data Sample Undefined Pause Resume SYNC100 WR48-63 Disc MBS Other HEC Data
0 184 0 0 0 0 0 0 0 0 0 0 0
1 115458 62 0 0 0 0 31 31 0 0 0 0
2 444 105932150 0 0 36 36 18754 18754 0 105894570 0 0
3 39683483 106817705 0 0 101 101 21752 21752 1037793 105736206 0 0
4 154 0 0 0 0 0 0 0 0 0 0 0
5 4413391 1852 0 0 0 0 926 926 0 0 0 0
6 586 0 0 0 0 0 0 0 0 0 0 0
7 4948958 2044 0 0 0 0 1022 1022 0 0 0 0
8 489 0 0 0 0 0 0 0 0 0 0 0
9 77 0 0 0 0 0 0 0 0 0 0 0
10 1172 0 0 0 0 0 0 0 0 0 0 0
11 163 2 0 0 0 0 1 1 0 0 0 0
12 754 0 0 0 0 0 0 0 0 0 0 0
13 94 0 0 0 0 0 0 0 0 0 0 0
14 605 0 0 0 0 0 0 0 0 0 0 0
15 177 0 0 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0 0 0 0
*** Timewarps
FEE ADC Pause Resume SYNC100 WR48-63 Disc MBS Undefined Samples
0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0
5 0 0 0 0 0 0 0 0 0
6 0 0 0 0 0 0 0 0 0
7 0 0 0 0 0 0 0 0 0
8 0 0 0 0 0 0 0 0 0
9 0 0 0 0 0 0 0 0 0
10 0 0 0 0 0 0 0 0 0
11 0 0 0 0 0 0 0 0 0
12 0 0 0 0 0 0 0 0 0
13 0 0 0 0 0 0 0 0 0
14 0 0 0 0 0 0 0 0 0
15 0 0 0 0 0 0 0 0 0
16 0 0 0 0 0 0 0 0 0
17 0 0 0 0 0 0 0 0 0
18 0 0 0 0 0 0 0 0 0
19 0 0 0 0 0 0 0 0 0
20 0 0 0 0 0 0 0 0 0
21 0 0 0 0 0 0 0 0 0
22 0 0 0 0 0 0 0 0 0
23 0 0 0 0 0 0 0 0 0
24 0 0 0 0 0 0 0 0 0
25 0 0 0 0 0 0 0 0 0
26 0 0 0 0 0 0 0 0 0
27 0 0 0 0 0 0 0 0 0
28 0 0 0 0 0 0 0 0 0
29 0 0 0 0 0 0 0 0 0
30 0 0 0 0 0 0 0 0 0
31 0 0 0 0 0 0 0 0 0
32 0 0 0 0 0 0 0 0 0
*** Program elapsed time: 29.027s ( 1102.409 blocks/s, 68.901 Mb/s)
|