source: mainline/tools/mkfat.py@ 3e72e41

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 3e72e41 was 7751ff1, checked in by Martin Decky <martin@…>, 13 years ago

improve FAT16 LFN creation support

  • add proper support for non-ASCII characters (by removing them from the legacy 8+3 names and by proper UTF-16 encoding in LFN names)
  • do not use the ugly global 8+3 disambiguation list, but create the legacy 8+3 names on directory basis
  • improve cstyle and code readability
  • Property mode set to 100755
File size: 15.5 KB
Line 
1#!/usr/bin/env python
2#
3# Copyright (c) 2008 Martin Decky
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions
8# are met:
9#
10# - Redistributions of source code must retain the above copyright
11# notice, this list of conditions and the following disclaimer.
12# - Redistributions in binary form must reproduce the above copyright
13# notice, this list of conditions and the following disclaimer in the
14# documentation and/or other materials provided with the distribution.
15# - The name of the author may not be used to endorse or promote products
16# derived from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28#
29
30"""
31FAT creator
32"""
33
34import sys
35import os
36import random
37import xstruct
38import array
39from imgutil import *
40
41def subtree_size(root, cluster_size, dirent_size):
42 "Recursive directory walk and calculate size"
43
44 size = 0
45 files = 2
46
47 for item in listdir_items(root):
48 if item.is_file:
49 size += align_up(item.size, cluster_size)
50 files += 1
51 elif item.is_dir:
52 size += subtree_size(item.path, cluster_size, dirent_size)
53 files += 1
54
55 return size + align_up(files * dirent_size, cluster_size)
56
57def root_entries(root):
58 "Return number of root directory entries"
59
60 return len(os.listdir(root))
61
62def write_file(item, outf, cluster_size, data_start, fat, reserved_clusters):
63 "Store the contents of a file"
64
65 prev = -1
66 first = 0
67
68 for data in chunks(item, cluster_size):
69 empty_cluster = fat.index(0)
70 fat[empty_cluster] = 0xffff
71
72 if (prev != -1):
73 fat[prev] = empty_cluster
74 else:
75 first = empty_cluster
76
77 prev = empty_cluster
78
79 outf.seek(data_start + (empty_cluster - reserved_clusters) * cluster_size)
80 outf.write(data)
81
82 return first, item.size
83
84def write_directory(directory, outf, cluster_size, data_start, fat, reserved_clusters, dirent_size, empty_cluster):
85 "Store the contents of a directory"
86
87 length = len(directory)
88 size = length * dirent_size
89 prev = -1
90 first = 0
91
92 i = 0
93 rd = 0;
94 while (rd < size):
95 if (prev != -1):
96 empty_cluster = fat.index(0)
97 fat[empty_cluster] = 0xffff
98 fat[prev] = empty_cluster
99 else:
100 first = empty_cluster
101
102 prev = empty_cluster
103
104 data = bytes()
105 data_len = 0
106 while ((i < length) and (data_len < cluster_size)):
107 if (i == 0):
108 directory[i].cluster = empty_cluster
109
110 data += directory[i].pack()
111 data_len += dirent_size
112 i += 1
113
114 outf.seek(data_start + (empty_cluster - reserved_clusters) * cluster_size)
115 outf.write(data)
116 rd += len(data)
117
118 return first, size
119
120DIR_ENTRY = """little:
121 char name[8] /* file name */
122 char ext[3] /* file extension */
123 uint8_t attr /* file attributes */
124 uint8_t lcase /* file name case (NT extension) */
125 uint8_t ctime_fine /* create time (fine resolution) */
126 uint16_t ctime /* create time */
127 uint16_t cdate /* create date */
128 uint16_t adate /* access date */
129 padding[2] /* EA-index */
130 uint16_t mtime /* modification time */
131 uint16_t mdate /* modification date */
132 uint16_t cluster /* first cluster */
133 uint32_t size /* file size */
134"""
135
136DOT_DIR_ENTRY = """little:
137 uint8_t signature /* 0x2e signature */
138 char name[7] /* empty */
139 char ext[3] /* empty */
140 uint8_t attr /* file attributes */
141 padding[1] /* reserved for NT */
142 uint8_t ctime_fine /* create time (fine resolution) */
143 uint16_t ctime /* create time */
144 uint16_t cdate /* create date */
145 uint16_t adate /* access date */
146 padding[2] /* EA-index */
147 uint16_t mtime /* modification time */
148 uint16_t mdate /* modification date */
149 uint16_t cluster /* first cluster */
150 uint32_t size /* file size */
151"""
152
153DOTDOT_DIR_ENTRY = """little:
154 uint8_t signature[2] /* 0x2e signature */
155 char name[6] /* empty */
156 char ext[3] /* empty */
157 uint8_t attr /* file attributes */
158 padding[1] /* reserved for NT */
159 uint8_t ctime_fine /* create time (fine resolution) */
160 uint16_t ctime /* create time */
161 uint16_t cdate /* create date */
162 uint16_t adate /* access date */
163 padding[2] /* EA-index */
164 uint16_t mtime /* modification time */
165 uint16_t mdate /* modification date */
166 uint16_t cluster /* first cluster */
167 uint32_t size /* file size */
168"""
169
170LFN_DIR_ENTRY = """little:
171 uint8_t seq /* sequence number */
172 char name1[10] /* first part of the name */
173 uint8_t attr /* attributes */
174 uint8_t rec_type /* LFN record type */
175 uint8_t checksum /* LFN checksum */
176 char name2[12] /* second part of the name */
177 uint16_t cluster /* cluster */
178 char name3[4] /* third part of the name */
179"""
180
181lchars = set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
182 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
183 'U', 'V', 'W', 'X', 'Y', 'Z',
184 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
185 '!', '#', '$', '%', '&', '\'', '(', ')', '-', '@',
186 '^', '_', '`', '{', '}', '~', '.'])
187
188def fat_lchars(name):
189 "Filter FAT legal characters"
190
191 filtered_name = ''
192
193 for char in name.encode('ascii', 'ignore').upper():
194 if not char in lchars:
195 continue
196
197 filtered_name += char
198
199 return filtered_name
200
201def fat_name83(name, name83_list):
202 "Create a 8.3 name for the given name"
203
204 ascii_name = fat_lchars(name)
205 ascii_parts = ascii_name.split('.')
206
207 short_name = ''
208 short_ext = ''
209 lfn = False
210
211 if len(ascii_name) > 11:
212 lfn = True
213
214 if len(ascii_parts) > 0:
215 short_name = ascii_parts[0]
216 if len(short_name) > 8:
217 lfn = True
218
219 if len(ascii_parts) > 1:
220 short_ext = ascii_parts[-1]
221 if len(short_ext) > 3:
222 lfn = True
223
224 if len(ascii_parts) > 2:
225 lfn = True
226
227 if lfn == False:
228 name83_list.append(short_name + '.' + short_ext)
229 return (short_name.ljust(8)[0:8], short_ext.ljust(3)[0:3], False)
230
231 # For filenames with multiple extensions, we treat the last one
232 # as the actual extension. The rest of the filename is stripped
233 # of dots and concatenated to form the short name
234 for part in ascii_parts[1:-1]:
235 short_name += part
236
237 for number in range(1, 999999):
238 number_str = ('~' + str(number)).upper()
239
240 if len(short_name) + len(number_str) > 8:
241 short_name = short_name[0:8 - len(number_str)]
242
243 short_name += number_str;
244
245 if not (short_name + '.' + short_ext) in name83_list:
246 break
247
248 name83_list.append(short_name + '.' + short_ext)
249 return (short_name.ljust(8)[0:8], short_ext.ljust(3)[0:3], True)
250
251def create_lfn_dirent(name, seq, checksum):
252 "Create LFN directory entry"
253
254 entry = xstruct.create(LFN_DIR_ENTRY)
255 name_rest = name[26:]
256
257 if len(name_rest) > 0:
258 entry.seq = seq
259 else:
260 entry.seq = seq | 0x40
261
262 entry.name1 = name[0:10]
263 entry.name2 = name[10:22]
264 entry.name3 = name[22:26]
265
266 entry.attr = 0x0F
267 entry.rec_type = 0
268 entry.checksum = checksum
269 entry.cluster = 0
270
271 return (entry, name_rest)
272
273def lfn_checksum(name):
274 "Calculate LFN checksum"
275
276 checksum = 0
277 for i in range(0, 11):
278 checksum = (((checksum & 1) << 7) + (checksum >> 1) + ord(name[i])) & 0xFF
279
280 return checksum
281
282def create_dirent(name, name83_list, directory, cluster, size):
283 short_name, short_ext, lfn = fat_name83(name, name83_list)
284
285 dir_entry = xstruct.create(DIR_ENTRY)
286
287 dir_entry.name = short_name
288 dir_entry.ext = short_ext
289
290 if (directory):
291 dir_entry.attr = 0x30
292 else:
293 dir_entry.attr = 0x20
294
295 dir_entry.lcase = 0x18
296 dir_entry.ctime_fine = 0 # FIXME
297 dir_entry.ctime = 0 # FIXME
298 dir_entry.cdate = 0 # FIXME
299 dir_entry.adate = 0 # FIXME
300 dir_entry.mtime = 0 # FIXME
301 dir_entry.mdate = 0 # FIXME
302 dir_entry.cluster = cluster
303
304 if (directory):
305 dir_entry.size = 0
306 else:
307 dir_entry.size = size
308
309 if not lfn:
310 return [dir_entry]
311
312 long_name = name.encode('utf_16_le')
313 entries = [dir_entry]
314
315 seq = 1
316 checksum = lfn_checksum(dir_entry.name + dir_entry.ext)
317
318 while len(long_name) > 0:
319 long_entry, long_name = create_lfn_dirent(long_name, seq, checksum)
320 entries.append(long_entry)
321 seq += 1
322
323 entries.reverse()
324 return entries
325
326def create_dot_dirent(empty_cluster):
327 dir_entry = xstruct.create(DOT_DIR_ENTRY)
328
329 dir_entry.signature = 0x2e
330 dir_entry.name = b' '
331 dir_entry.ext = b' '
332 dir_entry.attr = 0x10
333
334 dir_entry.ctime_fine = 0 # FIXME
335 dir_entry.ctime = 0 # FIXME
336 dir_entry.cdate = 0 # FIXME
337 dir_entry.adate = 0 # FIXME
338 dir_entry.mtime = 0 # FIXME
339 dir_entry.mdate = 0 # FIXME
340 dir_entry.cluster = empty_cluster
341 dir_entry.size = 0
342
343 return dir_entry
344
345def create_dotdot_dirent(parent_cluster):
346 dir_entry = xstruct.create(DOTDOT_DIR_ENTRY)
347
348 dir_entry.signature = [0x2e, 0x2e]
349 dir_entry.name = b' '
350 dir_entry.ext = b' '
351 dir_entry.attr = 0x10
352
353 dir_entry.ctime_fine = 0 # FIXME
354 dir_entry.ctime = 0 # FIXME
355 dir_entry.cdate = 0 # FIXME
356 dir_entry.adate = 0 # FIXME
357 dir_entry.mtime = 0 # FIXME
358 dir_entry.mdate = 0 # FIXME
359 dir_entry.cluster = parent_cluster
360 dir_entry.size = 0
361
362 return dir_entry
363
364def recursion(head, root, outf, cluster_size, root_start, data_start, fat, reserved_clusters, dirent_size, parent_cluster):
365 "Recursive directory walk"
366
367 directory = []
368 name83_list = []
369
370 if not head:
371 # Directory cluster preallocation
372 empty_cluster = fat.index(0)
373 fat[empty_cluster] = 0xFFFF
374
375 directory.append(create_dot_dirent(empty_cluster))
376 directory.append(create_dotdot_dirent(parent_cluster))
377 else:
378 empty_cluster = 0
379
380 for item in listdir_items(root):
381 if item.is_file:
382 rv = write_file(item, outf, cluster_size, data_start, fat, reserved_clusters)
383 directory.extend(create_dirent(item.name, name83_list, False, rv[0], rv[1]))
384 elif item.is_dir:
385 rv = recursion(False, item.path, outf, cluster_size, root_start, data_start, fat, reserved_clusters, dirent_size, empty_cluster)
386 directory.extend(create_dirent(item.name, name83_list, True, rv[0], rv[1]))
387
388 if head:
389 outf.seek(root_start)
390 for dir_entry in directory:
391 outf.write(dir_entry.pack())
392 else:
393 return write_directory(directory, outf, cluster_size, data_start, fat, reserved_clusters, dirent_size, empty_cluster)
394
395BOOT_SECTOR = """little:
396 uint8_t jmp[3] /* jump instruction */
397 char oem[8] /* OEM string */
398 uint16_t sector /* bytes per sector */
399 uint8_t cluster /* sectors per cluster */
400 uint16_t reserved /* reserved sectors */
401 uint8_t fats /* number of FATs */
402 uint16_t rootdir /* root directory entries */
403 uint16_t sectors /* total number of sectors */
404 uint8_t descriptor /* media descriptor */
405 uint16_t fat_sectors /* sectors per single FAT */
406 uint16_t track_sectors /* sectors per track */
407 uint16_t heads /* number of heads */
408 uint32_t hidden /* hidden sectors */
409 uint32_t sectors_big /* total number of sectors (if sectors == 0) */
410
411 /* Extended BIOS Parameter Block */
412 uint8_t drive /* physical drive number */
413 padding[1] /* reserved (current head) */
414 uint8_t extboot_signature /* extended boot signature */
415 uint32_t serial /* serial number */
416 char label[11] /* volume label */
417 char fstype[8] /* filesystem type */
418 padding[448] /* boot code */
419 uint8_t boot_signature[2] /* boot signature */
420"""
421
422EMPTY_SECTOR = """little:
423 padding[512] /* empty sector data */
424"""
425
426FAT_ENTRY = """little:
427 uint16_t next /* FAT16 entry */
428"""
429
430def usage(prname):
431 "Print usage syntax"
432 print(prname + " <EXTRA_BYTES> <PATH> <IMAGE>")
433
434def main():
435 if (len(sys.argv) < 4):
436 usage(sys.argv[0])
437 return
438
439 if (not sys.argv[1].isdigit()):
440 print("<EXTRA_BYTES> must be a number")
441 return
442
443 extra_bytes = int(sys.argv[1])
444
445 path = os.path.abspath(sys.argv[2].decode())
446 if (not os.path.isdir(path)):
447 print("<PATH> must be a directory")
448 return
449
450 fat16_clusters = 4096
451
452 sector_size = 512
453 cluster_size = 4096
454 dirent_size = 32
455 fatent_size = 2
456 fat_count = 2
457 reserved_clusters = 2
458
459 # Make sure the filesystem is large enough for FAT16
460 size = subtree_size(path, cluster_size, dirent_size) + reserved_clusters * cluster_size + extra_bytes
461 while (size // cluster_size < fat16_clusters):
462 if (cluster_size > sector_size):
463 cluster_size = cluster_size // 2
464 size = subtree_size(path, cluster_size, dirent_size) + reserved_clusters * cluster_size + extra_bytes
465 else:
466 size = fat16_clusters * cluster_size + reserved_clusters * cluster_size
467
468 root_size = align_up(root_entries(path) * dirent_size, cluster_size)
469
470 fat_size = align_up(align_up(size, cluster_size) // cluster_size * fatent_size, sector_size)
471
472 sectors = (cluster_size + fat_count * fat_size + root_size + size) // sector_size
473 root_start = cluster_size + fat_count * fat_size
474 data_start = root_start + root_size
475
476 outf = open(sys.argv[3], "wb")
477
478 boot_sector = xstruct.create(BOOT_SECTOR)
479 boot_sector.jmp = [0xEB, 0x3C, 0x90]
480 boot_sector.oem = b'MSDOS5.0'
481 boot_sector.sector = sector_size
482 boot_sector.cluster = cluster_size // sector_size
483 boot_sector.reserved = cluster_size // sector_size
484 boot_sector.fats = fat_count
485 boot_sector.rootdir = root_size // dirent_size
486 if (sectors <= 65535):
487 boot_sector.sectors = sectors
488 else:
489 boot_sector.sectors = 0
490 boot_sector.descriptor = 0xF8
491 boot_sector.fat_sectors = fat_size // sector_size
492 boot_sector.track_sectors = 63
493 boot_sector.heads = 6
494 boot_sector.hidden = 0
495 if (sectors > 65535):
496 boot_sector.sectors_big = sectors
497 else:
498 boot_sector.sectors_big = 0
499
500 boot_sector.drive = 0x80
501 boot_sector.extboot_signature = 0x29
502 boot_sector.serial = random.randint(0, 0x7fffffff)
503 boot_sector.label = b'HELENOS'
504 boot_sector.fstype = b'FAT16 '
505 boot_sector.boot_signature = [0x55, 0xAA]
506
507 outf.write(boot_sector.pack())
508
509 empty_sector = xstruct.create(EMPTY_SECTOR)
510
511 # Reserved sectors
512 for i in range(1, cluster_size // sector_size):
513 outf.write(empty_sector.pack())
514
515 # FAT tables
516 for i in range(0, fat_count):
517 for j in range(0, fat_size // sector_size):
518 outf.write(empty_sector.pack())
519
520 # Root directory
521 for i in range(0, root_size // sector_size):
522 outf.write(empty_sector.pack())
523
524 # Data
525 for i in range(0, size // sector_size):
526 outf.write(empty_sector.pack())
527
528 fat = array.array('L', [0] * (fat_size // fatent_size))
529 fat[0] = 0xfff8
530 fat[1] = 0xffff
531
532 recursion(True, path, outf, cluster_size, root_start, data_start, fat, reserved_clusters, dirent_size, 0)
533
534 # Store FAT
535 fat_entry = xstruct.create(FAT_ENTRY)
536 for i in range(0, fat_count):
537 outf.seek(cluster_size + i * fat_size)
538 for j in range(0, fat_size // fatent_size):
539 fat_entry.next = fat[j]
540 outf.write(fat_entry.pack())
541
542 outf.close()
543
544if __name__ == '__main__':
545 main()
Note: See TracBrowser for help on using the repository browser.