Before any heads explode... we are told that black level subtraction has not been done. It's the raw data (apart from the edge pixel garbage). "Does not require compensation" is in the eye of beholder, but it is what it is.blindbloom wrote:Therefore, I am proceeding with the understanding (hope!) that black level has already been applied or that the data was captured in a way that does not require this compensation.
Hm, I did investigate. When I dump the header, I see only hex zero at about byte 68, regardless of vertical or horizontal flips.6by9 wrote: I have stated many times that the Bayer order is present in the raw header, but nobody seems to be bothered to investigate. Try about byte 68 into the raw header. The enum values are even in the userland repo https://github.com/raspberrypi/userland ... pes.h#L144 Process that correctly and you can then even deal with deliberate flips.
Code: Select all
#!/bin/bash
RAWBLOCKSIZE=6404096
HEADERSIZE=32768
infile="$1"
offset="${2:-0}"
length="${3:-$HEADERSIZE}"
filesize=$(stat -c "%s" "$infile")
let start=filesize-RAWBLOCKSIZE+offset
dd if="$infile" iflag=skip_bytes,count_bytes \
skip=$start count=$length 2>/dev/null | hexdump -C
I corrected myself:bablokb wrote:Hm, I did investigate. When I dump the header, I see only hex zero at about byte 68, regardless of vertical or horizontal flips.
I use the following script to dump the header (you can pass the filename and optionally an offset into the header. As a third parameter you can pass the length, which defaults to the complete header size).
blindbloom had picked up on this at viewtopic.php?f=43&t=44918&start=150#p773622 and wasn't complaining with an offset of 244 (though his code only handles 2 of the 4 options), so 244 is probably right. I haven't checked in an actual file.Sorry, I gave a duff offset before for the Bayer order - the structure I was looking at was embedded within another. It's more likely to be around byte 244 into the header.
I also checked my files, and I see byte 242 (offset 241) as a candidate:6by9 wrote: I corrected myself:
viewtopic.php?f=43&t=44918&start=150#p773451blindbloom had picked up on this at viewtopic.php?f=43&t=44918&start=150#p773622 and wasn't complaining with an offset of 244 (though his code only handles 2 of the 4 options), so 244 is probably right. I haven't checked in an actual file.Sorry, I gave a duff offset before for the Bayer order - the structure I was looking at was embedded within another. It's more likely to be around byte 244 into the header.
edit: checked actual files and something isn't right there, so apologies. This does stir recollections of an issue we had before, so I'll investigate.
Code: Select all
let start=filesize-RAWBLOCKSIZE-1+offset
Code: Select all
uint8_t name[32]; /**< short text description of the mode, e.g. "720P" */
uint16_t width; /**< width of frame in pixels */
uint16_t height; /**< height of frame in pixels */
uint16_t padding_right; /**< additional non-data pixels to the right of the frame */
uint16_t padding_down; /**< additional non-data pixels below the frame */
uint32_t
uint32_t
uint32_t
uint32_t
uint32_t
uint16_t
uint16_t
uint16_t transform; /**< Possible transformations in this mode / user requested transformations */
uint16_t format; /**< Image format of the frame. */
uint8_t bayer_order; /**< If a Bayer format frame, the Bayer order in the frame. */
uint8_t bayer_format; /**< If a Bayer format frame, the packing mode of the data. */
Code: Select all
typedef enum
{ //defined to be identical to register bits
VC_IMAGE_BAYER_RGGB = 0,
VC_IMAGE_BAYER_GBRG = 1,
VC_IMAGE_BAYER_BGGR = 2,
VC_IMAGE_BAYER_GRBG = 3
} VC_IMAGE_BAYER_ORDER_T;
looks highly plausible.SW-11720: Write correct camera mode from write_raw_md_stage
Write the camera mode passed in switch_mode and not camera
calibration metadata.
Code: Select all
pi@raspberrypi ~ $ ./raw_header.sh no_flip.jpg 244 16
00000000 01 03 0a 00 00 00 00 00
pi@raspberrypi ~ $ ./raw_header.sh h_flip.jpg 244 16
00000000 02 03 0a 00 00 00 00 00 02 00 00 00 10 00 00 00 |................|
pi@raspberrypi ~ $ ./raw_header.sh v_flip.jpg 244 16
00000000 00 03 0a 00 00 00 00 00 02 00 00 00 10 00 00 00 |................|
pi@raspberrypi ~ $ ./raw_header.sh hv_flip.jpg 244 16
00000000 03 03 0a 00 00 00 00 00 02 00 00 00 10 00 00 00 |................|
Code: Select all
RawDecodeExample()
{
static const char rawFilename[] = "D:/RaspberryPi/RAW-GMB-Nov2014.jpg"; // from jbeale's site
//static const char rawFilename[] = "D:/RaspberryPi/RAW0029.jpg"; // from jbeale's site
static const int knSizeX = 2592;
static const int knSizeY = 1944;
static const int knEndOfRowPadding = 24; // padding at end of each row
static const int knBytesPerSet = 5;
static const int knPixelsPerSet = 4;
static const int knStrideY = knBytesPerSet * knSizeX / knPixelsPerSet + knEndOfRowPadding; // in bytes
static const int knHeaderSize = 32768;
static const int knEndOfImagePadding = 8 * knStrideY;
static const int knRawDataSize = knHeaderSize + knSizeY * knStrideY + knEndOfImagePadding;
// offsets into RAW header, relative to its start
static const int knIdOffset = 0; // char[4]
static const int knSensorDataOffset = 8; // uint32_t
static const int knVersionOffset = 16; // char[128?]
static const int knNameOffset = 176; // char[32]
static const int knWidthOffset = 208; // uint16_t
static const int knHeightOffset = 210; // uint16_t
static const int knPaddingRightOffset = 212; // uint16_t
static const int knPaddingDownOffset = 214; // uint16_t
static const int knTransformOffset = 240; // uint16_t
static const int knImageFormatOffset = 242; // uint16_t
static const int knBayerOrderOffset = 244; // BYTE
static const int knBayerFormatOffset = 245; // BYTE
static const int knXOffset = 420; // char[4]
// open the raw file
FILE* fd = NULL;
fopen_s(&fd, rawFilename, "rb");
// seek to the header
fseek(fd, -knRawDataSize, SEEK_END);
// read the beginning of the header
static const int knPartialHeaderSize = 512;
BYTE auHeader[knPartialHeaderSize];
fread(auHeader, sizeof(BYTE), sizeof(auHeader), fd);
// get the offset to the sensor data
UINT uOffset =
(static_cast<UINT>(auHeader[knSensorDataOffset + 3]) << 24) +
(static_cast<UINT>(auHeader[knSensorDataOffset + 2]) << 16) +
(static_cast<UINT>(auHeader[knSensorDataOffset + 1]) << 8) +
(static_cast<UINT>(auHeader[knSensorDataOffset]));
// extract encoded metadata
int nEncodedWidth = (static_cast<int>(auHeader[knWidthOffset + 1]) << 8) + static_cast<int>(auHeader[knWidthOffset]);
int nEncodedHeight = (static_cast<int>(auHeader[knHeightOffset + 1]) << 8) + static_cast<int>(auHeader[knHeightOffset]);
int nEncodedPaddingRight = (static_cast<int>(auHeader[knPaddingRightOffset + 1]) << 8) + static_cast<int>(auHeader[knPaddingRightOffset]);
int nEncodedPaddingDown = (static_cast<int>(auHeader[knPaddingDownOffset + 1]) << 8) + static_cast<int>(auHeader[knPaddingDownOffset]);
int nEncodedTransform = (static_cast<int>(auHeader[knTransformOffset + 1]) << 8) + static_cast<int>(auHeader[knTransformOffset]);
int nEncodedImageFormat = (static_cast<int>(auHeader[knImageFormatOffset + 1]) << 8) + static_cast<int>(auHeader[knImageFormatOffset]);
BYTE uBayerOrder = auHeader[knBayerOrderOffset];
BYTE uBayerFormat = auHeader[knBayerFormatOffset];
// make sure the header looks ok, as far as we're able to check it
assert(uOffset == knHeaderSize - 4); // offset - 4 of sensor data, relative to the start of the header
assert(strncmp(reinterpret_cast<const char*>(&auHeader[knIdOffset]), "BRCM", 4) == 0);
assert(strncmp(reinterpret_cast<const char*>(&auHeader[knVersionOffset]), "ov5647 version 0.1", 18) == 0);
assert(strncmp(reinterpret_cast<const char*>(&auHeader[knNameOffset]), "2592x1944", 9) == 0);
assert(strncmp(reinterpret_cast<const char*>(&auHeader[knXOffset]), "xxxx", 4) == 0);
assert(nEncodedWidth == knSizeX);
assert(nEncodedHeight == knSizeY);
assert(uBayerOrder <= 3);
// print some metadata
fprintf(stdout, "Image Width = %d\n", nEncodedWidth);
fprintf(stdout, "Image Height = %d\n", nEncodedHeight);
fprintf(stdout, "Padding Right = %d\n", nEncodedPaddingRight);
fprintf(stdout, "Padding Down = %d\n", nEncodedPaddingDown);
fprintf(stdout, "Transform = %d (0x%04x)\n", nEncodedTransform, nEncodedTransform);
fprintf(stdout, "Image Format = %d (0x%04x)\n", nEncodedImageFormat, nEncodedImageFormat);
fprintf(stdout, "Bayer Order = %s\n",
(uBayerOrder == 0) ? "RG/GB" :
(uBayerOrder == 1) ? "GB/RG" : // new firmware
(uBayerOrder == 2) ? "BG/GR" : // old firmware
(uBayerOrder == 3) ? "GR/BG" : "unrecognized");
fprintf(stdout, "Bayer Format = %d (0x%02x)\n", uBayerFormat, uBayerFormat);
fflush(stdout);
// seek to the sensor data
fseek(fd, uOffset + 4 - knPartialHeaderSize, SEEK_CUR);
// create an image for the Bayer data
GenImage image(knSizeX, knSizeY, keGray);
for (int nY = 0; nY < knSizeY; nY++)
{
for (int nX = 0; nX < knSizeX;)
{
BYTE auData[knBytesPerSet]; // every set of five bytes encodes four, 10-bit pixels (5 x 8 bits = 4 * 10 bits = 40 bits)
fread(auData, 1, knBytesPerSet, fd);
// compute the 10-bit sensor values, in the range [0, 1023]
UINT uV0 = (static_cast<UINT>(auData[0]) << 2) + (static_cast<UINT>(auData[4] & 0x03) >> 0);
UINT uV1 = (static_cast<UINT>(auData[1]) << 2) + (static_cast<UINT>(auData[4] & 0x0C) >> 2);
UINT uV2 = (static_cast<UINT>(auData[2]) << 2) + (static_cast<UINT>(auData[4] & 0x30) >> 4);
UINT uV3 = (static_cast<UINT>(auData[3]) << 2) + (static_cast<UINT>(auData[4] & 0xC0) >> 6);
// write the sensor values, scaling from the range [0, 1023] to [0.0, 1.0]
image.Write(nX++, nY, uV0 / 1023.0f);
image.Write(nX++, nY, uV1 / 1023.0f);
image.Write(nX++, nY, uV2 / 1023.0f);
image.Write(nX++, nY, uV3 / 1023.0f);
}
// seek to the first active pixel of the next row
fseek(fd, knEndOfRowPadding, SEEK_CUR);
}
// save the active sensor image
image.SaveToFile(L"D:/RaspberryPi/Bayer.png", ke16BitsPerChannel);
fclose(fd);
return;
}
Code: Select all
RAW-GMB-Nov2014.jpg (jbeale)
Off Dec Hx Binary ASC DataType VariableName Comment
--- --- -- -------- --- -------- -------------- ---------------------
0 66 42 01000010 'B'
1 82 52 01010010 'R'
2 67 43 01000011 'C'
3 77 4d 01001101 'M'
4 110 6e 01101110 'n'
5 0 00 00000000
6 0 00 00000000
7 0 00 00000000
8 252 fc 11111100
9 127 7f 01111111
10 0 00 00000000
11 0 00 00000000
12 0 00 00000000
13 0 00 00000000
14 0 00 00000000
15 0 00 00000000
16 111 6f 01101111 'o'
17 118 76 01110110 'v'
18 53 35 00110101 '5'
19 54 36 00110110 '6'
20 52 34 00110100 '4'
21 55 37 00110111 '7'
22 32 20 00100000 ' '
23 118 76 01110110 'v'
24 101 65 01100101 'e'
25 114 72 01110010 'r'
26 115 73 01110011 's'
27 105 69 01101001 'i'
28 111 6f 01101111 'o'
29 110 6e 01101110 'n'
30 32 20 00100000 ' '
31 48 30 00110000 '0'
32 46 2e 00101110 '.'
33 49 31 00110001 '1'
34 0 00 00000000
35 0 00 00000000
36 0 00 00000000
37 0 00 00000000
38 0 00 00000000
39 0 00 00000000
40 0 00 00000000
41 0 00 00000000
42 0 00 00000000
43 0 00 00000000
44 0 00 00000000
45 0 00 00000000
46 0 00 00000000
47 0 00 00000000
48 0 00 00000000
49 0 00 00000000
50 0 00 00000000
51 0 00 00000000
52 0 00 00000000
53 0 00 00000000
54 0 00 00000000
55 0 00 00000000
56 0 00 00000000
57 0 00 00000000
58 0 00 00000000
59 0 00 00000000
60 0 00 00000000
61 0 00 00000000
62 0 00 00000000
63 0 00 00000000
64 0 00 00000000
65 0 00 00000000
66 0 00 00000000
67 0 00 00000000
68 0 00 00000000
69 0 00 00000000
70 0 00 00000000
71 0 00 00000000
72 0 00 00000000
73 0 00 00000000
74 0 00 00000000
75 0 00 00000000
76 0 00 00000000
77 0 00 00000000
78 0 00 00000000
79 0 00 00000000
80 0 00 00000000
81 0 00 00000000
82 0 00 00000000
83 0 00 00000000
84 0 00 00000000
85 0 00 00000000
86 0 00 00000000
87 0 00 00000000
88 0 00 00000000
89 0 00 00000000
90 0 00 00000000
91 0 00 00000000
92 0 00 00000000
93 0 00 00000000
94 0 00 00000000
95 0 00 00000000
96 0 00 00000000
97 0 00 00000000
98 0 00 00000000
99 0 00 00000000
100 0 00 00000000
101 0 00 00000000
102 0 00 00000000
103 0 00 00000000
104 0 00 00000000
105 0 00 00000000
106 0 00 00000000
107 0 00 00000000
108 0 00 00000000
109 0 00 00000000
110 0 00 00000000
111 0 00 00000000
112 0 00 00000000
113 0 00 00000000
114 0 00 00000000
115 0 00 00000000
116 0 00 00000000
117 0 00 00000000
118 0 00 00000000
119 0 00 00000000
120 0 00 00000000
121 0 00 00000000
122 0 00 00000000
123 0 00 00000000
124 0 00 00000000
125 0 00 00000000
126 0 00 00000000
127 0 00 00000000
128 0 00 00000000
129 0 00 00000000
130 0 00 00000000
131 0 00 00000000
132 0 00 00000000
133 0 00 00000000
134 0 00 00000000
135 0 00 00000000
136 0 00 00000000
137 0 00 00000000
138 0 00 00000000
139 0 00 00000000
140 0 00 00000000
141 0 00 00000000
142 0 00 00000000
143 0 00 00000000
144 3 03 00000011
145 246 f6 11110110
146 0 00 00000000
147 0 00 00000000
148 224 e0 11100000
149 1 01 00000001
150 0 00 00000000
151 1 01 00000001
152 255 ff 11111111
153 0 00 00000000
154 0 00 00000000
155 0 00 00000000
156 0 00 00000000
157 0 00 00000000
158 0 00 00000000
159 0 00 00000000
160 192 c0 11000000
161 12 0c 00001100
162 0 00 00000000
163 0 00 00000000
164 0 00 00000000
165 0 00 00000000
166 0 00 00000000
167 0 00 00000000
168 0 00 00000000
169 0 00 00000000
170 0 00 00000000
171 0 00 00000000
172 0 00 00000000
173 0 00 00000000
174 0 00 00000000
175 0 00 00000000
176 50 32 00110010 '2' uint8_t name[32] short text description of the mode
177 53 35 00110101 '5'
178 57 39 00111001 '9'
179 50 32 00110010 '2'
180 120 78 01111000 'x'
181 49 31 00110001 '1'
182 57 39 00111001 '9'
183 52 34 00110100 '4'
184 52 34 00110100 '4'
185 0 00 00000000
186 0 00 00000000
187 0 00 00000000
188 0 00 00000000
189 0 00 00000000
190 0 00 00000000
191 0 00 00000000
192 0 00 00000000
193 0 00 00000000
194 0 00 00000000
195 0 00 00000000
196 0 00 00000000
197 0 00 00000000
198 0 00 00000000
199 0 00 00000000
200 0 00 00000000
201 0 00 00000000
202 0 00 00000000
203 0 00 00000000
204 0 00 00000000
205 0 00 00000000
206 0 00 00000000
207 0 00 00000000
208 32 20 00100000 ' ' uint16_t width width of frame in pixels (low byte, high byte)
209 10 0a 00001010
210 152 98 10011000 uint16_t height height of frame in pixels (low byte, high byte)
211 7 07 00000111
212 0 00 00000000 uint16_t padding_right additional non-data pixels to the right of the frame
213 0 00 00000000
214 0 00 00000000 uint16_t padding_down additional non-data pixels below the frame
215 0 00 00000000
216 27 1b 00011011 uint32_t
217 127 7f 01111111
218 0 00 00000000
219 0 00 00000000
220 176 b0 10110000 uint32_t
221 7 07 00000111
222 0 00 00000000
223 0 00 00000000
224 128 80 10000000 uint32_t
225 76 4c 01001100 'L'
226 206 ce 11001110
227 0 00 00000000
228 0 00 00000000 uint32_t
229 1 01 00000001
230 0 00 00000000
231 0 00 00000000
232 0 00 00000000 uint32_t
233 15 0f 00001111
234 0 00 00000000
235 0 00 00000000
236 1 01 00000001 uint16_t
237 0 00 00000000
238 1 01 00000001 uint16_t
239 0 00 00000000
240 0 00 00000000 uint16_t transform possible transformations in this mode / user requested transformations
241 0 00 00000000
242 33 21 00100001 '!' uint16_t format image format of the frame
243 0 00 00000000
244 1 01 00000001 uint8_t bayer_order if a Bayer format frame, the Bayer order in the frame
245 3 03 00000011 uint8_t bayer_format if a Bayer format frame, the Bayer format in the frame
246 10 0a 00001010
247 0 00 00000000
248 0 00 00000000
249 0 00 00000000
250 0 00 00000000
251 0 00 00000000
252 2 02 00000010
253 0 00 00000000
254 0 00 00000000
255 0 00 00000000
Code: Select all
t = Tiff('filename','r');
offsets = getTag(t,'SubIFD');
setSubDirectory(t,offsets(1));
cfa = read(t);
close(t);
figure(1);
histogram(cfa(:),0:4000);