Index: uspace/srv/bd/hr/hr.c =================================================================== --- uspace/srv/bd/hr/hr.c (revision 40be7eb4cdef8c2565a90c877426276916ce26f1) +++ uspace/srv/bd/hr/hr.c (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -1,4 +1,4 @@ /* - * Copyright (c) 2024 Miroslav Cimerman + * Copyright (c) 2025 Miroslav Cimerman * All rights reserved. * @@ -49,4 +49,6 @@ #include +#include "fge.h" +#include "io.h" #include "superblock.h" #include "util.h" @@ -83,4 +85,5 @@ list_foreach(hr_volumes, lvolumes, hr_volume_t, vol) { if (vol->svc_id == svc_id) { + hr_fpool_destroy(vol->fge); hr_fini_devs(vol); list_remove(&vol->lvolumes); @@ -154,4 +157,13 @@ return; } + + hr_fpool_t *fge = hr_fpool_create(16, 32, sizeof(hr_io_t)); + if (fge == NULL) { + free(new_volume); + free(cfg); + async_answer_0(icall, ENOMEM); + return; + } + new_volume->fge = fge; str_cpy(new_volume->devname, HR_DEVNAME_LEN, cfg->devname); @@ -234,5 +246,11 @@ } - fibril_mutex_initialize(&new_volume->lock); + fibril_mutex_initialize(&new_volume->lock); /* XXX: will remove this */ + + fibril_mutex_initialize(&new_volume->halt_lock); + new_volume->halt_please = false; + + fibril_rwlock_initialize(&new_volume->extents_lock); + fibril_rwlock_initialize(&new_volume->states_lock); list_initialize(&new_volume->range_lock_list); @@ -260,4 +278,5 @@ error: free(cfg); + free(fge); hr_fini_devs(new_volume); free(new_volume); Index: uspace/srv/bd/hr/io.c =================================================================== --- uspace/srv/bd/hr/io.c (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) +++ uspace/srv/bd/hr/io.c (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2025 Miroslav Cimerman + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup hr + * @{ + */ +/** + * @file + */ + +#include +#include +#include +#include +#include + +#include "io.h" +#include "util.h" +#include "var.h" + +errno_t hr_io_worker(void *arg) +{ + hr_io_t *io = arg; + hr_extent_t *extents = (hr_extent_t *)&io->vol->extents; + size_t ext_idx = io->extent; + errno_t rc; + + HR_DEBUG("WORKER on extent: %lu, ba: %lu, cnt: %lu\n", + io->extent, io->ba, io->cnt); + + switch (io->type) { + case HR_BD_SYNC: + rc = block_sync_cache(extents[ext_idx].svc_id, + io->ba, io->cnt); + if (rc == ENOTSUP) + rc = EOK; + break; + case HR_BD_READ: + rc = block_read_direct(extents[ext_idx].svc_id, io->ba, + io->cnt, io->data_read); + break; + case HR_BD_WRITE: + rc = block_write_direct(extents[ext_idx].svc_id, io->ba, + io->cnt, io->data_write); + break; + default: + return EINVAL; + } + + if (rc != EOK) + io->state_callback(io->vol, io->extent, rc); + + HR_DEBUG("WORKER rc: %s\n", str_error(rc)); + + return rc; +} + +/** @} + */ Index: uspace/srv/bd/hr/io.h =================================================================== --- uspace/srv/bd/hr/io.h (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) +++ uspace/srv/bd/hr/io.h (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2025 Miroslav Cimerman + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup hr + * @{ + */ +/** + * @file + */ + +#ifndef _HR_IO_H +#define _HR_IO_H + +#include "var.h" + +typedef struct hr_io { + hr_bd_op_type_t type; + uint64_t ba; + uint64_t cnt; + size_t extent; + void *data_read; + const void *data_write; + hr_volume_t *vol; + void (*state_callback)(hr_volume_t *, size_t, errno_t); +} hr_io_t; + +errno_t hr_io_worker(void *); + +#endif + +/** @} + */ Index: uspace/srv/bd/hr/meson.build =================================================================== --- uspace/srv/bd/hr/meson.build (revision 40be7eb4cdef8c2565a90c877426276916ce26f1) +++ uspace/srv/bd/hr/meson.build (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -1,4 +1,4 @@ # -# Copyright (c) 2024 Miroslav Cimerman +# Copyright (c) 2025 Miroslav Cimerman # All rights reserved. # @@ -31,4 +31,5 @@ 'fge.c', 'hr.c', + 'io.c', 'raid0.c', 'raid1.c', Index: uspace/srv/bd/hr/raid0.c =================================================================== --- uspace/srv/bd/hr/raid0.c (revision 40be7eb4cdef8c2565a90c877426276916ce26f1) +++ uspace/srv/bd/hr/raid0.c (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -1,4 +1,4 @@ /* - * Copyright (c) 2024 Miroslav Cimerman + * Copyright (c) 2025 Miroslav Cimerman * All rights reserved. * @@ -48,4 +48,5 @@ #include +#include "io.h" #include "superblock.h" #include "util.h" @@ -54,5 +55,4 @@ extern loc_srv_t *hr_srv; -static errno_t hr_raid0_check_vol_status(hr_volume_t *); static errno_t hr_raid0_update_vol_status(hr_volume_t *); static errno_t hr_raid0_bd_op(hr_bd_op_type_t, bd_srv_t *, aoff64_t, size_t, @@ -95,4 +95,6 @@ return rc; + hr_update_vol_status(new_volume, HR_VOL_ONLINE); + bd_srvs_init(&new_volume->hr_bds); new_volume->hr_bds.ops = &hr_raid0_bd_ops; @@ -127,7 +129,5 @@ void hr_raid0_status_event(hr_volume_t *vol) { - fibril_mutex_lock(&vol->lock); (void)hr_raid0_update_vol_status(vol); - fibril_mutex_unlock(&vol->lock); } @@ -175,11 +175,4 @@ *rnb = vol->data_blkno; return EOK; -} - -static errno_t hr_raid0_check_vol_status(hr_volume_t *vol) -{ - if (vol->status == HR_VOL_ONLINE) - return EOK; - return EIO; } @@ -190,18 +183,40 @@ static errno_t hr_raid0_update_vol_status(hr_volume_t *vol) { + fibril_rwlock_read_lock(&vol->states_lock); hr_vol_status_t old_state = vol->status; for (size_t i = 0; i < vol->extent_no; i++) { if (vol->extents[i].status != HR_EXT_ONLINE) { + fibril_rwlock_read_unlock(&vol->states_lock); + fibril_rwlock_write_lock(&vol->states_lock); if (old_state != HR_VOL_FAULTY) hr_update_vol_status(vol, HR_VOL_FAULTY); + fibril_rwlock_write_unlock(&vol->states_lock); return EIO; } } - - if (old_state != HR_VOL_ONLINE) - hr_update_vol_status(vol, HR_VOL_ONLINE); - - return EOK; + fibril_rwlock_read_unlock(&vol->states_lock); + + return EOK; +} + +static void raid0_state_callback(hr_volume_t *vol, size_t extent, errno_t rc) +{ + if (rc == EOK) + return; + + fibril_rwlock_write_lock(&vol->states_lock); + + switch (rc) { + case ENOENT: + hr_update_ext_status(vol, extent, HR_EXT_MISSING); + break; + default: + hr_update_ext_status(vol, extent, HR_EXT_FAILED); + } + + hr_update_vol_status(vol, HR_VOL_FAULTY); + + fibril_rwlock_write_unlock(&vol->states_lock); } @@ -216,9 +231,32 @@ uint8_t *data_read = dst; + fibril_rwlock_read_lock(&vol->states_lock); + if (vol->status != HR_VOL_ONLINE) { + fibril_rwlock_read_unlock(&vol->states_lock); + return EIO; + } + fibril_rwlock_read_unlock(&vol->states_lock); + /* propagate sync */ if (type == HR_BD_SYNC && ba == 0 && cnt == 0) { - hr_sync_all_extents(vol); - rc = hr_raid0_update_vol_status(vol); - return rc; + hr_fgroup_t *group = hr_fgroup_create(vol->fge, vol->extent_no); + + for (size_t i = 0; i < vol->extent_no; i++) { + hr_io_t *io = hr_fgroup_alloc(group); + io->extent = i; + io->ba = ba; + io->cnt = cnt; + io->type = type; + io->vol = vol; + io->state_callback = raid0_state_callback; + + hr_fgroup_submit(group, hr_io_worker, io); + } + + size_t bad; + (void)hr_fgroup_wait(group, NULL, &bad); + if (bad > 0) + return EIO; + return EOK; } @@ -232,55 +270,39 @@ uint64_t strip_size = vol->strip_size / vol->bsize; /* in blocks */ - uint64_t stripe = ba / strip_size; /* stripe number */ - uint64_t extent = stripe % vol->extent_no; - uint64_t ext_stripe = stripe / vol->extent_no; /* stripe level */ - uint64_t strip_off = ba % strip_size; /* strip offset */ - - fibril_mutex_lock(&vol->lock); - - rc = hr_raid0_check_vol_status(vol); - if (rc != EOK) { - fibril_mutex_unlock(&vol->lock); - return EIO; - } + uint64_t strip_no = ba / strip_size; + uint64_t extent = strip_no % vol->extent_no; + uint64_t stripe = strip_no / vol->extent_no; + uint64_t strip_off = ba % strip_size; left = cnt; + /* calculate how many strips does the IO span */ + size_t end_strip_no = (ba + cnt - 1) / strip_size; + size_t span = end_strip_no - strip_no + 1; + + hr_fgroup_t *group = hr_fgroup_create(vol->fge, span); + while (left != 0) { - phys_block = ext_stripe * strip_size + strip_off; + phys_block = stripe * strip_size + strip_off; cnt = min(left, strip_size - strip_off); len = vol->bsize * cnt; hr_add_ba_offset(vol, &phys_block); - switch (type) { - case HR_BD_SYNC: - rc = block_sync_cache(vol->extents[extent].svc_id, - phys_block, cnt); - /* allow unsupported sync */ - if (rc == ENOTSUP) - rc = EOK; - break; - case HR_BD_READ: - rc = block_read_direct(vol->extents[extent].svc_id, - phys_block, cnt, data_read); + + hr_io_t *io = hr_fgroup_alloc(group); + io->extent = extent; + io->data_write = data_write; + io->data_read = data_read; + io->ba = ba; + io->cnt = cnt; + io->type = type; + io->vol = vol; + io->state_callback = raid0_state_callback; + + hr_fgroup_submit(group, hr_io_worker, io); + + if (type == HR_BD_READ) data_read += len; - break; - case HR_BD_WRITE: - rc = block_write_direct(vol->extents[extent].svc_id, - phys_block, cnt, data_write); + else if (type == HR_BD_WRITE) data_write += len; - break; - default: - rc = EINVAL; - } - - if (rc == ENOENT) { - hr_update_ext_status(vol, extent, HR_EXT_MISSING); - rc = EIO; - goto error; - } else if (rc != EOK) { - hr_update_ext_status(vol, extent, HR_EXT_FAILED); - rc = EIO; - goto error; - } left -= cnt; @@ -288,13 +310,15 @@ extent++; if (extent >= vol->extent_no) { - ext_stripe++; + stripe++; extent = 0; } } -error: - (void)hr_raid0_update_vol_status(vol); - fibril_mutex_unlock(&vol->lock); - return rc; + size_t bad; + (void)hr_fgroup_wait(group, NULL, &bad); + if (bad > 0) + return EIO; + + return EOK; } Index: uspace/srv/bd/hr/util.h =================================================================== --- uspace/srv/bd/hr/util.h (revision 40be7eb4cdef8c2565a90c877426276916ce26f1) +++ uspace/srv/bd/hr/util.h (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -1,4 +1,4 @@ /* - * Copyright (c) 2024 Miroslav Cimerman + * Copyright (c) 2025 Miroslav Cimerman * All rights reserved. * @@ -38,4 +38,5 @@ #include +#include #include "var.h" Index: uspace/srv/bd/hr/var.h =================================================================== --- uspace/srv/bd/hr/var.h (revision 40be7eb4cdef8c2565a90c877426276916ce26f1) +++ uspace/srv/bd/hr/var.h (revision bc3d695b6287b7807d133734f5b8790bdcb16cb4) @@ -1,4 +1,4 @@ /* - * Copyright (c) 2024 Miroslav Cimerman + * Copyright (c) 2025 Miroslav Cimerman * All rights reserved. * @@ -43,4 +43,6 @@ #include +#include "fge.h" + #define NAME "hr" @@ -60,5 +62,10 @@ bd_srvs_t hr_bds; - link_t lvolumes; + link_t lvolumes; /* protected by static hr_volumes_lock in hr.c */ + + /* + * XXX: will be gone after all paralelization, but still used + * in yet-unparallelized levels + */ fibril_mutex_t lock; @@ -66,10 +73,8 @@ fibril_mutex_t range_lock_list_lock; + hr_fpool_t *fge; + + /* after assembly, these are invariant */ size_t extent_no; - hr_extent_t extents[HR_MAX_EXTENTS]; - - size_t hotspare_no; - hr_extent_t hotspares[HR_MAX_HOTSPARES]; - size_t bsize; uint64_t nblocks; @@ -77,14 +82,26 @@ uint64_t data_offset; /* in blocks */ uint32_t strip_size; + hr_level_t level; + uint8_t layout; /* RAID Level Qualifier */ + service_id_t svc_id; + char devname[HR_DEVNAME_LEN]; + + hr_extent_t extents[HR_MAX_EXTENTS]; + size_t hotspare_no; + hr_extent_t hotspares[HR_MAX_HOTSPARES]; + + /* protects ordering (hr_extent_t.svc_id, hotspares) */ + fibril_rwlock_t extents_lock; + + /* protects states (hr_extent_t.status, hr_vol_status_t.status) */ + fibril_rwlock_t states_lock; + + /* for halting IO requests when a REBUILD start waits */ + bool halt_please; + fibril_mutex_t halt_lock; uint64_t rebuild_blk; - uint64_t counter; /* metadata syncing */ - - service_id_t svc_id; hr_vol_status_t status; - hr_level_t level; - uint8_t layout; /* RAID Level Qualifier */ - char devname[HR_DEVNAME_LEN]; } hr_volume_t;