source: mainline/kernel/generic/src/mm/frame.c@ 4b241f3

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 4b241f3 was e49e234, checked in by Martin Decky <martin@…>, 17 years ago

kernel memory management revisited (phase 2): map physical memory according to zones

  • ia32: register reserved and ACPI zones
  • pareas are now used only for mapping of present physical memory (hw_area() is gone)
  • firmware zones and physical addresses outside any zones are allowed to be mapped generally
  • fix nasty antient bug in zones_insert_zone()
  • Property mode set to 100644
File size: 33.9 KB
Line 
1/*
2 * Copyright (c) 2001-2005 Jakub Jermar
3 * Copyright (c) 2005 Sergey Bondari
4 * Copyright (c) 2009 Martin Decky
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * - Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * - Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * - The name of the author may not be used to endorse or promote products
17 * derived from this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31/** @addtogroup genericmm
32 * @{
33 */
34
35/**
36 * @file
37 * @brief Physical frame allocator.
38 *
39 * This file contains the physical frame allocator and memory zone management.
40 * The frame allocator is built on top of the buddy allocator.
41 *
42 * @see buddy.c
43 */
44
45#include <arch/types.h>
46#include <mm/frame.h>
47#include <mm/as.h>
48#include <panic.h>
49#include <debug.h>
50#include <adt/list.h>
51#include <synch/mutex.h>
52#include <synch/condvar.h>
53#include <arch/asm.h>
54#include <arch.h>
55#include <print.h>
56#include <align.h>
57#include <mm/slab.h>
58#include <bitops.h>
59#include <macros.h>
60#include <config.h>
61
62zones_t zones;
63
64/*
65 * Synchronization primitives used to sleep when there is no memory
66 * available.
67 */
68mutex_t mem_avail_mtx;
69condvar_t mem_avail_cv;
70count_t mem_avail_req = 0; /**< Number of frames requested. */
71count_t mem_avail_gen = 0; /**< Generation counter. */
72
73/********************/
74/* Helper functions */
75/********************/
76
77static inline index_t frame_index(zone_t *zone, frame_t *frame)
78{
79 return (index_t) (frame - zone->frames);
80}
81
82static inline index_t frame_index_abs(zone_t *zone, frame_t *frame)
83{
84 return (index_t) (frame - zone->frames) + zone->base;
85}
86
87static inline bool frame_index_valid(zone_t *zone, index_t index)
88{
89 return (index < zone->count);
90}
91
92static inline index_t make_frame_index(zone_t *zone, frame_t *frame)
93{
94 return (frame - zone->frames);
95}
96
97/** Initialize frame structure.
98 *
99 * @param frame Frame structure to be initialized.
100 *
101 */
102static void frame_initialize(frame_t *frame)
103{
104 frame->refcount = 1;
105 frame->buddy_order = 0;
106}
107
108/*******************/
109/* Zones functions */
110/*******************/
111
112/** Insert-sort zone into zones list.
113 *
114 * Assume interrupts are disabled and zones lock is
115 * locked.
116 *
117 * @param base Base frame of the newly inserted zone.
118 * @param count Number of frames of the newly inserted zone.
119 *
120 * @return Zone number on success, -1 on error.
121 *
122 */
123static count_t zones_insert_zone(pfn_t base, count_t count)
124{
125 if (zones.count + 1 == ZONES_MAX) {
126 printf("Maximum zone count %u exceeded!\n", ZONES_MAX);
127 return (count_t) -1;
128 }
129
130 count_t i;
131 for (i = 0; i < zones.count; i++) {
132 /* Check for overlap */
133 if (overlaps(base, count,
134 zones.info[i].base, zones.info[i].count)) {
135 printf("Zones overlap!\n");
136 return (count_t) -1;
137 }
138 if (base < zones.info[i].base)
139 break;
140 }
141
142 /* Move other zones up */
143 count_t j;
144 for (j = zones.count; j > i; j--) {
145 zones.info[j] = zones.info[j - 1];
146 zones.info[j].buddy_system->data =
147 (void *) &zones.info[j - 1];
148 }
149
150 zones.count++;
151
152 return i;
153}
154
155/** Get total available frames.
156 *
157 * Assume interrupts are disabled and zones lock is
158 * locked.
159 *
160 * @return Total number of available frames.
161 *
162 */
163static count_t total_frames_free(void)
164{
165 count_t total = 0;
166 count_t i;
167 for (i = 0; i < zones.count; i++)
168 total += zones.info[i].free_count;
169
170 return total;
171}
172
173/** Find a zone with a given frames.
174 *
175 * Assume interrupts are disabled and zones lock is
176 * locked.
177 *
178 * @param frame Frame number contained in zone.
179 * @param count Number of frames to look for.
180 * @param hint Used as zone hint.
181 *
182 * @return Zone index or -1 if not found.
183 *
184 */
185count_t find_zone(pfn_t frame, count_t count, count_t hint)
186{
187 if (hint >= zones.count)
188 hint = 0;
189
190 count_t i = hint;
191 do {
192 if ((zones.info[i].base <= frame)
193 && (zones.info[i].base + zones.info[i].count >= frame + count))
194 return i;
195
196 i++;
197 if (i >= zones.count)
198 i = 0;
199 } while (i != hint);
200
201 return (count_t) -1;
202}
203
204/** @return True if zone can allocate specified order */
205static bool zone_can_alloc(zone_t *zone, uint8_t order)
206{
207 return (zone_flags_available(zone->flags)
208 && buddy_system_can_alloc(zone->buddy_system, order));
209}
210
211/** Find a zone that can allocate order frames.
212 *
213 * Assume interrupts are disabled and zones lock is
214 * locked.
215 *
216 * @param order Size (2^order) of free space we are trying to find.
217 * @param flags Required flags of the target zone.
218 * @param hind Preferred zone.
219 *
220 */
221static count_t find_free_zone(uint8_t order, zone_flags_t flags, count_t hint)
222{
223 if (hint >= zones.count)
224 hint = 0;
225
226 count_t i = hint;
227 do {
228 /*
229 * Check whether the zone meets the search criteria.
230 */
231 if ((zones.info[i].flags & flags) == flags) {
232 /*
233 * Check if the zone has 2^order frames area available.
234 */
235 if (zone_can_alloc(&zones.info[i], order))
236 return i;
237 }
238
239 i++;
240 if (i >= zones.count)
241 i = 0;
242 } while (i != hint);
243
244 return (count_t) -1;
245}
246
247/**************************/
248/* Buddy system functions */
249/**************************/
250
251/** Buddy system find_block implementation.
252 *
253 * Find block that is parent of current list.
254 * That means go to lower addresses, until such block is found
255 *
256 * @param order Order of parent must be different then this
257 * parameter!!
258 *
259 */
260static link_t *zone_buddy_find_block(buddy_system_t *buddy, link_t *child,
261 uint8_t order)
262{
263 frame_t *frame = list_get_instance(child, frame_t, buddy_link);
264 zone_t *zone = (zone_t *) buddy->data;
265
266 index_t index = frame_index(zone, frame);
267 do {
268 if (zone->frames[index].buddy_order != order)
269 return &zone->frames[index].buddy_link;
270 } while (index-- > 0);
271
272 return NULL;
273}
274
275/** Buddy system find_buddy implementation.
276 *
277 * @param buddy Buddy system.
278 * @param block Block for which buddy should be found.
279 *
280 * @return Buddy for given block if found.
281 *
282 */
283static link_t *zone_buddy_find_buddy(buddy_system_t *buddy, link_t *block)
284{
285 frame_t *frame = list_get_instance(block, frame_t, buddy_link);
286 zone_t *zone = (zone_t *) buddy->data;
287 ASSERT(IS_BUDDY_ORDER_OK(frame_index_abs(zone, frame),
288 frame->buddy_order));
289
290 bool is_left = IS_BUDDY_LEFT_BLOCK_ABS(zone, frame);
291 bool is_right = IS_BUDDY_RIGHT_BLOCK_ABS(zone, frame);
292
293 ASSERT(is_left ^ is_right);
294
295 index_t index;
296 if (is_left) {
297 index = (frame_index(zone, frame)) +
298 (1 << frame->buddy_order);
299 } else { /* if (is_right) */
300 index = (frame_index(zone, frame)) -
301 (1 << frame->buddy_order);
302 }
303
304 if (frame_index_valid(zone, index)) {
305 if ((zone->frames[index].buddy_order == frame->buddy_order) &&
306 (zone->frames[index].refcount == 0)) {
307 return &zone->frames[index].buddy_link;
308 }
309 }
310
311 return NULL;
312}
313
314/** Buddy system bisect implementation.
315 *
316 * @param buddy Buddy system.
317 * @param block Block to bisect.
318 *
319 * @return Right block.
320 *
321 */
322static link_t *zone_buddy_bisect(buddy_system_t *buddy, link_t *block)
323{
324 frame_t *frame_l = list_get_instance(block, frame_t, buddy_link);
325 frame_t *frame_r = (frame_l + (1 << (frame_l->buddy_order - 1)));
326
327 return &frame_r->buddy_link;
328}
329
330/** Buddy system coalesce implementation.
331 *
332 * @param buddy Buddy system.
333 * @param block_1 First block.
334 * @param block_2 First block's buddy.
335 *
336 * @return Coalesced block (actually block that represents lower
337 * address).
338 *
339 */
340static link_t *zone_buddy_coalesce(buddy_system_t *buddy, link_t *block_1,
341 link_t *block_2)
342{
343 frame_t *frame1 = list_get_instance(block_1, frame_t, buddy_link);
344 frame_t *frame2 = list_get_instance(block_2, frame_t, buddy_link);
345
346 return ((frame1 < frame2) ? block_1 : block_2);
347}
348
349/** Buddy system set_order implementation.
350 *
351 * @param buddy Buddy system.
352 * @param block Buddy system block.
353 * @param order Order to set.
354 *
355 */
356static void zone_buddy_set_order(buddy_system_t *buddy, link_t *block,
357 uint8_t order)
358{
359 list_get_instance(block, frame_t, buddy_link)->buddy_order = order;
360}
361
362/** Buddy system get_order implementation.
363 *
364 * @param buddy Buddy system.
365 * @param block Buddy system block.
366 *
367 * @return Order of block.
368 *
369 */
370static uint8_t zone_buddy_get_order(buddy_system_t *buddy, link_t *block)
371{
372 return list_get_instance(block, frame_t, buddy_link)->buddy_order;
373}
374
375/** Buddy system mark_busy implementation.
376 *
377 * @param buddy Buddy system.
378 * @param block Buddy system block.
379 *
380 */
381static void zone_buddy_mark_busy(buddy_system_t *buddy, link_t * block)
382{
383 list_get_instance(block, frame_t, buddy_link)->refcount = 1;
384}
385
386/** Buddy system mark_available implementation.
387 *
388 * @param buddy Buddy system.
389 * @param block Buddy system block.
390 */
391static void zone_buddy_mark_available(buddy_system_t *buddy, link_t *block)
392{
393 list_get_instance(block, frame_t, buddy_link)->refcount = 0;
394}
395
396static buddy_system_operations_t zone_buddy_system_operations = {
397 .find_buddy = zone_buddy_find_buddy,
398 .bisect = zone_buddy_bisect,
399 .coalesce = zone_buddy_coalesce,
400 .set_order = zone_buddy_set_order,
401 .get_order = zone_buddy_get_order,
402 .mark_busy = zone_buddy_mark_busy,
403 .mark_available = zone_buddy_mark_available,
404 .find_block = zone_buddy_find_block
405};
406
407/******************/
408/* Zone functions */
409/******************/
410
411/** Allocate frame in particular zone.
412 *
413 * Assume zone is locked and is available for allocation.
414 * Panics if allocation is impossible.
415 *
416 * @param zone Zone to allocate from.
417 * @param order Allocate exactly 2^order frames.
418 *
419 * @return Frame index in zone.
420 *
421 */
422static pfn_t zone_frame_alloc(zone_t *zone, uint8_t order)
423{
424 ASSERT(zone_flags_available(zone->flags));
425
426 /* Allocate frames from zone buddy system */
427 link_t *link = buddy_system_alloc(zone->buddy_system, order);
428
429 ASSERT(link);
430
431 /* Update zone information. */
432 zone->free_count -= (1 << order);
433 zone->busy_count += (1 << order);
434
435 /* Frame will be actually a first frame of the block. */
436 frame_t *frame = list_get_instance(link, frame_t, buddy_link);
437
438 /* Get frame address */
439 return make_frame_index(zone, frame);
440}
441
442/** Free frame from zone.
443 *
444 * Assume zone is locked and is available for deallocation.
445 *
446 * @param zone Pointer to zone from which the frame is to be freed.
447 * @param frame_idx Frame index relative to zone.
448 *
449 */
450static void zone_frame_free(zone_t *zone, index_t frame_idx)
451{
452 ASSERT(zone_flags_available(zone->flags));
453
454 frame_t *frame = &zone->frames[frame_idx];
455
456 /* Remember frame order */
457 uint8_t order = frame->buddy_order;
458
459 ASSERT(frame->refcount);
460
461 if (!--frame->refcount) {
462 buddy_system_free(zone->buddy_system, &frame->buddy_link);
463
464 /* Update zone information. */
465 zone->free_count += (1 << order);
466 zone->busy_count -= (1 << order);
467 }
468}
469
470/** Return frame from zone. */
471static frame_t *zone_get_frame(zone_t *zone, index_t frame_idx)
472{
473 ASSERT(frame_idx < zone->count);
474 return &zone->frames[frame_idx];
475}
476
477/** Mark frame in zone unavailable to allocation. */
478static void zone_mark_unavailable(zone_t *zone, index_t frame_idx)
479{
480 ASSERT(zone_flags_available(zone->flags));
481
482 frame_t *frame = zone_get_frame(zone, frame_idx);
483 if (frame->refcount)
484 return;
485
486 link_t *link = buddy_system_alloc_block(zone->buddy_system,
487 &frame->buddy_link);
488
489 ASSERT(link);
490 zone->free_count--;
491}
492
493/** Merge two zones.
494 *
495 * Expect buddy to point to space at least zone_conf_size large.
496 * Assume z1 & z2 are locked and compatible and zones lock is
497 * locked.
498 *
499 * @param z1 First zone to merge.
500 * @param z2 Second zone to merge.
501 * @param old_z1 Original date of the first zone.
502 * @param buddy Merged zone buddy.
503 *
504 */
505static void zone_merge_internal(count_t z1, count_t z2, zone_t *old_z1, buddy_system_t *buddy)
506{
507 ASSERT(zone_flags_available(zones.info[z1].flags));
508 ASSERT(zone_flags_available(zones.info[z2].flags));
509 ASSERT(zones.info[z1].flags == zones.info[z2].flags);
510 ASSERT(zones.info[z1].base < zones.info[z2].base);
511 ASSERT(!overlaps(zones.info[z1].base, zones.info[z1].count,
512 zones.info[z2].base, zones.info[z2].count));
513
514 /* Difference between zone bases */
515 pfn_t base_diff = zones.info[z2].base - zones.info[z1].base;
516
517 zones.info[z1].count = base_diff + zones.info[z2].count;
518 zones.info[z1].free_count += zones.info[z2].free_count;
519 zones.info[z1].busy_count += zones.info[z2].busy_count;
520 zones.info[z1].buddy_system = buddy;
521
522 uint8_t order = fnzb(zones.info[z1].count);
523 buddy_system_create(zones.info[z1].buddy_system, order,
524 &zone_buddy_system_operations, (void *) &zones.info[z1]);
525
526 zones.info[z1].frames =
527 (frame_t *) ((uint8_t *) zones.info[z1].buddy_system
528 + buddy_conf_size(order));
529
530 /* This marks all frames busy */
531 count_t i;
532 for (i = 0; i < zones.info[z1].count; i++)
533 frame_initialize(&zones.info[z1].frames[i]);
534
535 /* Copy frames from both zones to preserve full frame orders,
536 * parents etc. Set all free frames with refcount = 0 to 1, because
537 * we add all free frames to buddy allocator later again, clearing
538 * order to 0. Don't set busy frames with refcount = 0, as they
539 * will not be reallocated during merge and it would make later
540 * problems with allocation/free.
541 */
542 for (i = 0; i < old_z1->count; i++)
543 zones.info[z1].frames[i] = old_z1->frames[i];
544
545 for (i = 0; i < zones.info[z2].count; i++)
546 zones.info[z1].frames[base_diff + i]
547 = zones.info[z2].frames[i];
548
549 i = 0;
550 while (i < zones.info[z1].count) {
551 if (zones.info[z1].frames[i].refcount) {
552 /* Skip busy frames */
553 i += 1 << zones.info[z1].frames[i].buddy_order;
554 } else {
555 /* Free frames, set refcount = 1
556 * (all free frames have refcount == 0, we need not
557 * to check the order)
558 */
559 zones.info[z1].frames[i].refcount = 1;
560 zones.info[z1].frames[i].buddy_order = 0;
561 i++;
562 }
563 }
564
565 /* Add free blocks from the original zone z1 */
566 while (zone_can_alloc(old_z1, 0)) {
567 /* Allocate from the original zone */
568 pfn_t frame_idx = zone_frame_alloc(old_z1, 0);
569
570 /* Free the frame from the merged zone */
571 frame_t *frame = &zones.info[z1].frames[frame_idx];
572 frame->refcount = 0;
573 buddy_system_free(zones.info[z1].buddy_system, &frame->buddy_link);
574 }
575
576 /* Add free blocks from the original zone z2 */
577 while (zone_can_alloc(&zones.info[z2], 0)) {
578 /* Allocate from the original zone */
579 pfn_t frame_idx = zone_frame_alloc(&zones.info[z2], 0);
580
581 /* Free the frame from the merged zone */
582 frame_t *frame = &zones.info[z1].frames[base_diff + frame_idx];
583 frame->refcount = 0;
584 buddy_system_free(zones.info[z1].buddy_system, &frame->buddy_link);
585 }
586}
587
588/** Return old configuration frames into the zone.
589 *
590 * We have two cases:
591 * - The configuration data is outside the zone
592 * -> do nothing (perhaps call frame_free?)
593 * - The configuration data was created by zone_create
594 * or updated by reduce_region -> free every frame
595 *
596 * @param znum The actual zone where freeing should occur.
597 * @param pfn Old zone configuration frame.
598 * @param count Old zone frame count.
599 *
600 */
601static void return_config_frames(count_t znum, pfn_t pfn, count_t count)
602{
603 ASSERT(zone_flags_available(zones.info[znum].flags));
604
605 count_t cframes = SIZE2FRAMES(zone_conf_size(count));
606
607 if ((pfn < zones.info[znum].base)
608 || (pfn >= zones.info[znum].base + zones.info[znum].count))
609 return;
610
611 frame_t *frame
612 = &zones.info[znum].frames[pfn - zones.info[znum].base];
613 ASSERT(!frame->buddy_order);
614
615 count_t i;
616 for (i = 0; i < cframes; i++) {
617 zones.info[znum].busy_count++;
618 zone_frame_free(&zones.info[znum],
619 pfn - zones.info[znum].base + i);
620 }
621}
622
623/** Reduce allocated block to count of order 0 frames.
624 *
625 * The allocated block needs 2^order frames. Reduce all frames
626 * in the block to order 0 and free the unneeded frames. This means that
627 * when freeing the previously allocated block starting with frame_idx,
628 * you have to free every frame.
629 *
630 * @param znum Zone.
631 * @param frame_idx Index the first frame of the block.
632 * @param count Allocated frames in block.
633 *
634 */
635static void zone_reduce_region(count_t znum, pfn_t frame_idx, count_t count)
636{
637 ASSERT(zone_flags_available(zones.info[znum].flags));
638 ASSERT(frame_idx + count < zones.info[znum].count);
639
640 uint8_t order = zones.info[znum].frames[frame_idx].buddy_order;
641 ASSERT((count_t) (1 << order) >= count);
642
643 /* Reduce all blocks to order 0 */
644 count_t i;
645 for (i = 0; i < (count_t) (1 << order); i++) {
646 frame_t *frame = &zones.info[znum].frames[i + frame_idx];
647 frame->buddy_order = 0;
648 if (!frame->refcount)
649 frame->refcount = 1;
650 ASSERT(frame->refcount == 1);
651 }
652
653 /* Free unneeded frames */
654 for (i = count; i < (count_t) (1 << order); i++)
655 zone_frame_free(&zones.info[znum], i + frame_idx);
656}
657
658/** Merge zones z1 and z2.
659 *
660 * The merged zones must be 2 zones with no zone existing in between
661 * (which means that z2 = z1 + 1). Both zones must be available zones
662 * with the same flags.
663 *
664 * When you create a new zone, the frame allocator configuration does
665 * not to be 2^order size. Once the allocator is running it is no longer
666 * possible, merged configuration data occupies more space :-/
667 *
668 * The function uses
669 *
670 */
671bool zone_merge(count_t z1, count_t z2)
672{
673 ipl_t ipl = interrupts_disable();
674 spinlock_lock(&zones.lock);
675
676 bool ret = true;
677
678 /* We can join only 2 zones with none existing inbetween,
679 * the zones have to be available and with the same
680 * set of flags
681 */
682 if ((z1 >= zones.count) || (z2 >= zones.count)
683 || (z2 - z1 != 1)
684 || (!zone_flags_available(zones.info[z1].flags))
685 || (!zone_flags_available(zones.info[z2].flags))
686 || (zones.info[z1].flags != zones.info[z2].flags)) {
687 ret = false;
688 goto errout;
689 }
690
691 pfn_t cframes = SIZE2FRAMES(zone_conf_size(
692 zones.info[z2].base - zones.info[z1].base
693 + zones.info[z2].count));
694
695 uint8_t order;
696 if (cframes == 1)
697 order = 0;
698 else
699 order = fnzb(cframes - 1) + 1;
700
701 /* Allocate merged zone data inside one of the zones */
702 pfn_t pfn;
703 if (zone_can_alloc(&zones.info[z1], order)) {
704 pfn = zones.info[z1].base + zone_frame_alloc(&zones.info[z1], order);
705 } else if (zone_can_alloc(&zones.info[z2], order)) {
706 pfn = zones.info[z2].base + zone_frame_alloc(&zones.info[z2], order);
707 } else {
708 ret = false;
709 goto errout;
710 }
711
712 /* Preserve original data from z1 */
713 zone_t old_z1 = zones.info[z1];
714 old_z1.buddy_system->data = (void *) &old_z1;
715
716 /* Do zone merging */
717 buddy_system_t *buddy = (buddy_system_t *) PA2KA(PFN2ADDR(pfn));
718 zone_merge_internal(z1, z2, &old_z1, buddy);
719
720 /* Free unneeded config frames */
721 zone_reduce_region(z1, pfn - zones.info[z1].base, cframes);
722
723 /* Subtract zone information from busy frames */
724 zones.info[z1].busy_count -= cframes;
725
726 /* Free old zone information */
727 return_config_frames(z1,
728 ADDR2PFN(KA2PA((uintptr_t) old_z1.frames)), old_z1.count);
729 return_config_frames(z1,
730 ADDR2PFN(KA2PA((uintptr_t) zones.info[z2].frames)),
731 zones.info[z2].count);
732
733 /* Move zones down */
734 count_t i;
735 for (i = z2 + 1; i < zones.count; i++) {
736 zones.info[i - 1] = zones.info[i];
737 zones.info[i - 1].buddy_system->data =
738 (void *) &zones.info[i - 1];
739 }
740
741 zones.count--;
742
743errout:
744 spinlock_unlock(&zones.lock);
745 interrupts_restore(ipl);
746
747 return ret;
748}
749
750/** Merge all mergeable zones into one big zone.
751 *
752 * It is reasonable to do this on systems where
753 * BIOS reports parts in chunks, so that we could
754 * have 1 zone (it's faster).
755 *
756 */
757void zone_merge_all(void)
758{
759 count_t i = 0;
760 while (i < zones.count) {
761 if (!zone_merge(i, i + 1))
762 i++;
763 }
764}
765
766/** Create new frame zone.
767 *
768 * @param zone Zone to construct.
769 * @param buddy Address of buddy system configuration information.
770 * @param start Physical address of the first frame within the zone.
771 * @param count Count of frames in zone.
772 * @param flags Zone flags.
773 *
774 * @return Initialized zone.
775 *
776 */
777static void zone_construct(zone_t *zone, buddy_system_t *buddy, pfn_t start, count_t count, zone_flags_t flags)
778{
779 zone->base = start;
780 zone->count = count;
781 zone->flags = flags;
782 zone->free_count = count;
783 zone->busy_count = 0;
784 zone->buddy_system = buddy;
785
786 if (zone_flags_available(flags)) {
787 /*
788 * Compute order for buddy system and initialize
789 */
790 uint8_t order = fnzb(count);
791 buddy_system_create(zone->buddy_system, order,
792 &zone_buddy_system_operations, (void *) zone);
793
794 /* Allocate frames _after_ the confframe */
795
796 /* Check sizes */
797 zone->frames = (frame_t *) ((uint8_t *) zone->buddy_system +
798 buddy_conf_size(order));
799
800 count_t i;
801 for (i = 0; i < count; i++)
802 frame_initialize(&zone->frames[i]);
803
804 /* Stuffing frames */
805 for (i = 0; i < count; i++) {
806 zone->frames[i].refcount = 0;
807 buddy_system_free(zone->buddy_system, &zone->frames[i].buddy_link);
808 }
809 } else
810 zone->frames = NULL;
811}
812
813/** Compute configuration data size for zone.
814 *
815 * @param count Size of zone in frames.
816 *
817 * @return Size of zone configuration info (in bytes).
818 *
819 */
820uintptr_t zone_conf_size(count_t count)
821{
822 return (count * sizeof(frame_t) + buddy_conf_size(fnzb(count)));
823}
824
825/** Create and add zone to system.
826 *
827 * @param start First frame number (absolute).
828 * @param count Size of zone in frames.
829 * @param confframe Where configuration frames are supposed to be.
830 * Automatically checks, that we will not disturb the
831 * kernel and possibly init. If confframe is given
832 * _outside_ this zone, it is expected, that the area is
833 * already marked BUSY and big enough to contain
834 * zone_conf_size() amount of data. If the confframe is
835 * inside the area, the zone free frame information is
836 * modified not to include it.
837 *
838 * @return Zone number or -1 on error.
839 *
840 */
841count_t zone_create(pfn_t start, count_t count, pfn_t confframe, zone_flags_t flags)
842{
843 ipl_t ipl = interrupts_disable();
844 spinlock_lock(&zones.lock);
845
846 if (zone_flags_available(flags)) { /* Create available zone */
847 /* Theoretically we could have NULL here, practically make sure
848 * nobody tries to do that. If some platform requires, remove
849 * the assert
850 */
851 ASSERT(confframe != NULL);
852
853 /* If confframe is supposed to be inside our zone, then make sure
854 * it does not span kernel & init
855 */
856 count_t confcount = SIZE2FRAMES(zone_conf_size(count));
857 if ((confframe >= start) && (confframe < start + count)) {
858 for (; confframe < start + count; confframe++) {
859 uintptr_t addr = PFN2ADDR(confframe);
860 if (overlaps(addr, PFN2ADDR(confcount),
861 KA2PA(config.base), config.kernel_size))
862 continue;
863
864 if (overlaps(addr, PFN2ADDR(confcount),
865 KA2PA(config.stack_base), config.stack_size))
866 continue;
867
868 bool overlap = false;
869 count_t i;
870 for (i = 0; i < init.cnt; i++)
871 if (overlaps(addr, PFN2ADDR(confcount),
872 KA2PA(init.tasks[i].addr),
873 init.tasks[i].size)) {
874 overlap = true;
875 break;
876 }
877 if (overlap)
878 continue;
879
880 break;
881 }
882
883 if (confframe >= start + count)
884 panic("Cannot find configuration data for zone.");
885 }
886
887 count_t znum = zones_insert_zone(start, count);
888 if (znum == (count_t) -1) {
889 spinlock_unlock(&zones.lock);
890 interrupts_restore(ipl);
891 return (count_t) -1;
892 }
893
894 buddy_system_t *buddy = (buddy_system_t *) PA2KA(PFN2ADDR(confframe));
895 zone_construct(&zones.info[znum], buddy, start, count, flags);
896
897 /* If confdata in zone, mark as unavailable */
898 if ((confframe >= start) && (confframe < start + count)) {
899 count_t i;
900 for (i = confframe; i < confframe + confcount; i++)
901 zone_mark_unavailable(&zones.info[znum],
902 i - zones.info[znum].base);
903 }
904
905 spinlock_unlock(&zones.lock);
906 interrupts_restore(ipl);
907
908 return znum;
909 }
910
911 /* Non-available zone */
912 count_t znum = zones_insert_zone(start, count);
913 if (znum == (count_t) -1) {
914 spinlock_unlock(&zones.lock);
915 interrupts_restore(ipl);
916 return (count_t) -1;
917 }
918 zone_construct(&zones.info[znum], NULL, start, count, flags);
919
920 spinlock_unlock(&zones.lock);
921 interrupts_restore(ipl);
922
923 return znum;
924}
925
926/*******************/
927/* Frame functions */
928/*******************/
929
930/** Set parent of frame. */
931void frame_set_parent(pfn_t pfn, void *data, count_t hint)
932{
933 ipl_t ipl = interrupts_disable();
934 spinlock_lock(&zones.lock);
935
936 count_t znum = find_zone(pfn, 1, hint);
937
938 ASSERT(znum != (count_t) -1);
939
940 zone_get_frame(&zones.info[znum],
941 pfn - zones.info[znum].base)->parent = data;
942
943 spinlock_unlock(&zones.lock);
944 interrupts_restore(ipl);
945}
946
947void *frame_get_parent(pfn_t pfn, count_t hint)
948{
949 ipl_t ipl = interrupts_disable();
950 spinlock_lock(&zones.lock);
951
952 count_t znum = find_zone(pfn, 1, hint);
953
954 ASSERT(znum != (count_t) -1);
955
956 void *res = zone_get_frame(&zones.info[znum],
957 pfn - zones.info[znum].base)->parent;
958
959 spinlock_unlock(&zones.lock);
960 interrupts_restore(ipl);
961
962 return res;
963}
964
965/** Allocate power-of-two frames of physical memory.
966 *
967 * @param order Allocate exactly 2^order frames.
968 * @param flags Flags for host zone selection and address processing.
969 * @param pzone Preferred zone.
970 *
971 * @return Physical address of the allocated frame.
972 *
973 */
974void *frame_alloc_generic(uint8_t order, frame_flags_t flags, count_t *pzone)
975{
976 count_t size = ((count_t) 1) << order;
977 ipl_t ipl;
978 count_t hint = pzone ? (*pzone) : 0;
979
980loop:
981 ipl = interrupts_disable();
982 spinlock_lock(&zones.lock);
983
984 /*
985 * First, find suitable frame zone.
986 */
987 count_t znum = find_free_zone(order,
988 FRAME_TO_ZONE_FLAGS(flags), hint);
989
990 /* If no memory, reclaim some slab memory,
991 if it does not help, reclaim all */
992 if ((znum == (count_t) -1) && (!(flags & FRAME_NO_RECLAIM))) {
993 count_t freed = slab_reclaim(0);
994
995 if (freed > 0)
996 znum = find_free_zone(order,
997 FRAME_TO_ZONE_FLAGS(flags), hint);
998
999 if (znum == (count_t) -1) {
1000 freed = slab_reclaim(SLAB_RECLAIM_ALL);
1001 if (freed > 0)
1002 znum = find_free_zone(order,
1003 FRAME_TO_ZONE_FLAGS(flags), hint);
1004 }
1005 }
1006
1007 if (znum == (count_t) -1) {
1008 if (flags & FRAME_ATOMIC) {
1009 spinlock_unlock(&zones.lock);
1010 interrupts_restore(ipl);
1011 return NULL;
1012 }
1013
1014#ifdef CONFIG_DEBUG
1015 count_t avail = total_frames_free();
1016#endif
1017
1018 spinlock_unlock(&zones.lock);
1019 interrupts_restore(ipl);
1020
1021 /*
1022 * Sleep until some frames are available again.
1023 */
1024
1025#ifdef CONFIG_DEBUG
1026 printf("Thread %" PRIu64 " waiting for %" PRIc " frames, "
1027 "%" PRIc " available.\n", THREAD->tid, size, avail);
1028#endif
1029
1030 mutex_lock(&mem_avail_mtx);
1031
1032 if (mem_avail_req > 0)
1033 mem_avail_req = min(mem_avail_req, size);
1034 else
1035 mem_avail_req = size;
1036 count_t gen = mem_avail_gen;
1037
1038 while (gen == mem_avail_gen)
1039 condvar_wait(&mem_avail_cv, &mem_avail_mtx);
1040
1041 mutex_unlock(&mem_avail_mtx);
1042
1043#ifdef CONFIG_DEBUG
1044 printf("Thread %" PRIu64 " woken up.\n", THREAD->tid);
1045#endif
1046
1047 goto loop;
1048 }
1049
1050 pfn_t pfn = zone_frame_alloc(&zones.info[znum], order)
1051 + zones.info[znum].base;
1052
1053 spinlock_unlock(&zones.lock);
1054 interrupts_restore(ipl);
1055
1056 if (pzone)
1057 *pzone = znum;
1058
1059 if (flags & FRAME_KA)
1060 return (void *) PA2KA(PFN2ADDR(pfn));
1061
1062 return (void *) PFN2ADDR(pfn);
1063}
1064
1065/** Free a frame.
1066 *
1067 * Find respective frame structure for supplied physical frame address.
1068 * Decrement frame reference count. If it drops to zero, move the frame
1069 * structure to free list.
1070 *
1071 * @param frame Physical Address of of the frame to be freed.
1072 *
1073 */
1074void frame_free(uintptr_t frame)
1075{
1076 ipl_t ipl = interrupts_disable();
1077 spinlock_lock(&zones.lock);
1078
1079 /*
1080 * First, find host frame zone for addr.
1081 */
1082 pfn_t pfn = ADDR2PFN(frame);
1083 count_t znum = find_zone(pfn, 1, NULL);
1084
1085 ASSERT(znum != (count_t) -1);
1086
1087 zone_frame_free(&zones.info[znum], pfn - zones.info[znum].base);
1088
1089 spinlock_unlock(&zones.lock);
1090 interrupts_restore(ipl);
1091
1092 /*
1093 * Signal that some memory has been freed.
1094 */
1095 mutex_lock(&mem_avail_mtx);
1096 if (mem_avail_req > 0)
1097 mem_avail_req--;
1098
1099 if (mem_avail_req == 0) {
1100 mem_avail_gen++;
1101 condvar_broadcast(&mem_avail_cv);
1102 }
1103 mutex_unlock(&mem_avail_mtx);
1104}
1105
1106/** Add reference to frame.
1107 *
1108 * Find respective frame structure for supplied PFN and
1109 * increment frame reference count.
1110 *
1111 * @param pfn Frame number of the frame to be freed.
1112 *
1113 */
1114void frame_reference_add(pfn_t pfn)
1115{
1116 ipl_t ipl = interrupts_disable();
1117 spinlock_lock(&zones.lock);
1118
1119 /*
1120 * First, find host frame zone for addr.
1121 */
1122 count_t znum = find_zone(pfn, 1, NULL);
1123
1124 ASSERT(znum != (count_t) -1);
1125
1126 zones.info[znum].frames[pfn - zones.info[znum].base].refcount++;
1127
1128 spinlock_unlock(&zones.lock);
1129 interrupts_restore(ipl);
1130}
1131
1132/** Mark given range unavailable in frame zones. */
1133void frame_mark_unavailable(pfn_t start, count_t count)
1134{
1135 ipl_t ipl = interrupts_disable();
1136 spinlock_lock(&zones.lock);
1137
1138 count_t i;
1139 for (i = 0; i < count; i++) {
1140 count_t znum = find_zone(start + i, 1, 0);
1141 if (znum == (count_t) -1) /* PFN not found */
1142 continue;
1143
1144 zone_mark_unavailable(&zones.info[znum],
1145 start + i - zones.info[znum].base);
1146 }
1147
1148 spinlock_unlock(&zones.lock);
1149 interrupts_restore(ipl);
1150}
1151
1152/** Initialize physical memory management. */
1153void frame_init(void)
1154{
1155 if (config.cpu_active == 1) {
1156 zones.count = 0;
1157 spinlock_initialize(&zones.lock, "zones.lock");
1158 mutex_initialize(&mem_avail_mtx, MUTEX_ACTIVE);
1159 condvar_initialize(&mem_avail_cv);
1160 }
1161
1162 /* Tell the architecture to create some memory */
1163 frame_arch_init();
1164 if (config.cpu_active == 1) {
1165 frame_mark_unavailable(ADDR2PFN(KA2PA(config.base)),
1166 SIZE2FRAMES(config.kernel_size));
1167 frame_mark_unavailable(ADDR2PFN(KA2PA(config.stack_base)),
1168 SIZE2FRAMES(config.stack_size));
1169
1170 count_t i;
1171 for (i = 0; i < init.cnt; i++) {
1172 pfn_t pfn = ADDR2PFN(KA2PA(init.tasks[i].addr));
1173 frame_mark_unavailable(pfn,
1174 SIZE2FRAMES(init.tasks[i].size));
1175 }
1176
1177 if (ballocs.size)
1178 frame_mark_unavailable(ADDR2PFN(KA2PA(ballocs.base)),
1179 SIZE2FRAMES(ballocs.size));
1180
1181 /* Black list first frame, as allocating NULL would
1182 * fail in some places
1183 */
1184 frame_mark_unavailable(0, 1);
1185 }
1186}
1187
1188/** Return total size of all zones. */
1189uint64_t zone_total_size(void)
1190{
1191 ipl_t ipl = interrupts_disable();
1192 spinlock_lock(&zones.lock);
1193
1194 uint64_t total = 0;
1195 count_t i;
1196 for (i = 0; i < zones.count; i++)
1197 total += (uint64_t) FRAMES2SIZE(zones.info[i].count);
1198
1199 spinlock_unlock(&zones.lock);
1200 interrupts_restore(ipl);
1201
1202 return total;
1203}
1204
1205/** Prints list of zones. */
1206void zone_print_list(void)
1207{
1208#ifdef __32_BITS__
1209 printf("# base address frames flags free frames busy frames\n");
1210 printf("-- ------------ ------------ -------- ------------ ------------\n");
1211#endif
1212
1213#ifdef __64_BITS__
1214 printf("# base address frames flags free frames busy frames\n");
1215 printf("-- -------------------- ------------ -------- ------------ ------------\n");
1216#endif
1217
1218 /*
1219 * Because printing may require allocation of memory, we may not hold
1220 * the frame allocator locks when printing zone statistics. Therefore,
1221 * we simply gather the statistics under the protection of the locks and
1222 * print the statistics when the locks have been released.
1223 *
1224 * When someone adds/removes zones while we are printing the statistics,
1225 * we may end up with inaccurate output (e.g. a zone being skipped from
1226 * the listing).
1227 */
1228
1229 count_t i;
1230 for (i = 0;; i++) {
1231 ipl_t ipl = interrupts_disable();
1232 spinlock_lock(&zones.lock);
1233
1234 if (i >= zones.count) {
1235 spinlock_unlock(&zones.lock);
1236 interrupts_restore(ipl);
1237 break;
1238 }
1239
1240 uintptr_t base = PFN2ADDR(zones.info[i].base);
1241 count_t count = zones.info[i].count;
1242 zone_flags_t flags = zones.info[i].flags;
1243 count_t free_count = zones.info[i].free_count;
1244 count_t busy_count = zones.info[i].busy_count;
1245
1246 spinlock_unlock(&zones.lock);
1247 interrupts_restore(ipl);
1248
1249 bool available = zone_flags_available(flags);
1250
1251 printf("%-2" PRIc, i);
1252
1253#ifdef __32_BITS__
1254 printf(" %10p", base);
1255#endif
1256
1257#ifdef __64_BITS__
1258 printf(" %18p", base);
1259#endif
1260
1261 printf(" %12" PRIc " %c%c%c ", count,
1262 available ? 'A' : ' ',
1263 (flags & ZONE_RESERVED) ? 'R' : ' ',
1264 (flags & ZONE_FIRMWARE) ? 'F' : ' ');
1265
1266 if (available)
1267 printf("%12" PRIc " %12" PRIc,
1268 free_count, busy_count);
1269
1270 printf("\n");
1271 }
1272}
1273
1274/** Prints zone details.
1275 *
1276 * @param num Zone base address or zone number.
1277 *
1278 */
1279void zone_print_one(count_t num)
1280{
1281 ipl_t ipl = interrupts_disable();
1282 spinlock_lock(&zones.lock);
1283 count_t znum = (count_t) -1;
1284
1285 count_t i;
1286 for (i = 0; i < zones.count; i++) {
1287 if ((i == num) || (PFN2ADDR(zones.info[i].base) == num)) {
1288 znum = i;
1289 break;
1290 }
1291 }
1292
1293 if (znum == (count_t) -1) {
1294 spinlock_unlock(&zones.lock);
1295 interrupts_restore(ipl);
1296 printf("Zone not found.\n");
1297 return;
1298 }
1299
1300 uintptr_t base = PFN2ADDR(zones.info[i].base);
1301 zone_flags_t flags = zones.info[i].flags;
1302 count_t count = zones.info[i].count;
1303 count_t free_count = zones.info[i].free_count;
1304 count_t busy_count = zones.info[i].busy_count;
1305
1306 spinlock_unlock(&zones.lock);
1307 interrupts_restore(ipl);
1308
1309 bool available = zone_flags_available(flags);
1310
1311 printf("Zone number: %" PRIc "\n", znum);
1312 printf("Zone base address: %p\n", base);
1313 printf("Zone size: %" PRIc " frames (%" PRIs " KiB)\n", count,
1314 SIZE2KB(FRAMES2SIZE(count)));
1315 printf("Zone flags: %c%c%c\n",
1316 available ? 'A' : ' ',
1317 (flags & ZONE_RESERVED) ? 'R' : ' ',
1318 (flags & ZONE_FIRMWARE) ? 'F' : ' ');
1319
1320 if (available) {
1321 printf("Allocated space: %" PRIc " frames (%" PRIs " KiB)\n",
1322 busy_count, SIZE2KB(FRAMES2SIZE(busy_count)));
1323 printf("Available space: %" PRIc " frames (%" PRIs " KiB)\n",
1324 free_count, SIZE2KB(FRAMES2SIZE(free_count)));
1325 }
1326}
1327
1328/** @}
1329 */
Note: See TracBrowser for help on using the repository browser.