Repository navigation
Expand file tree
/
Copy pathwd_sched.c
More file actions
2360 lines (2028 loc) · 64.6 KB
/
Copy pathwd_sched.c
File metadata and controls
2360 lines (2028 loc) · 64.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
/*
* Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved.
* Copyright 2020-2021 Linaro ltd.
*
* Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion
*
* Key improvements:
* - Single global hash table with (region_id, mode, op_type, prop) dimensions
* - Segment list for non-contiguous ctx ranges
* - Dual-domain queues for session key.
* - Dynamic ctx expansion in HUNGRY mode based on load threshold
* - Packet reception is handled through the active queues in the session key.
* - Simplified sched_init: only allocate one sync + one async ctx
* - Removed redundant wd_sched_info layer
*/
#define _GNU_SOURCE
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <sched.h>
#include <numa.h>
#include <limits.h>
#include <pthread.h>
#include "wd_sched.h"
#include "wd_alg.h"
#include "wd_internal.h"
#define MAX_POLL_TIMES 1000
#define SKEY_CTX_MAX_NUM 16
#define SKEY_MAX_THREAD_NUM 64
#define SKEY_LOAD_UPDATE_INTERVAL 1
#define HW_QUEUE_FULL_DEPTH 1024
/* ============================================================================
* Hash Table Configuration
* ============================================================================
*/
#define WD_SCHED_MAX_BUCKETS 512
#define WD_SCHED_MIN_BUCKETS 32
#define HASH_PRIME1 73
#define HASH_PRIME2 13
#define HASH_PRIME3 7
#define HASH_PRIME4 11
/* ============================================================================
* Scheduling Region Mode
* ============================================================================
*/
enum sched_region_mode {
SCHED_MODE_SYNC = 0,
SCHED_MODE_ASYNC = 1,
SCHED_MODE_BUTT
};
/* ============================================================================
* Segment List for Domain Index Organization
* ============================================================================
*/
/**
* wd_sched_ctx_segment - Contiguous segment of ctx indices in domain
* @begin: Start index of this segment
* @end: End index of this segment (inclusive)
* @next: Pointer to next segment in the linked list
*
* Supports non-contiguous ctx ranges via segment list.
*/
struct wd_sched_ctx_segment {
__u32 begin;
__u32 end;
struct wd_sched_ctx_segment *next;
};
/* ============================================================================
* Session key domain cache processing.
* ============================================================================
*/
/**
* wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains
*
* Design principles:
* - Fixed array for cache-friendly memory layout
* - Atomic operations for lock-free load tracking
* - Simple RR and load balancing strategies
* - Maximum 16 queues per thread (typical usage)
*/
struct wd_sched_domain_idx_cache {
/* Queue index array */
__u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */
__u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */
__u32 valid_count; /* Number of valid queues */
/* Scheduling state */
__u32 rr_ptr; /* Round-robin pointer */
__u32 min_load_idx; /* Cached min load index */
__u32 op_counter; /* Operation counter for updates */
__u8 load_decreased; /* Non-zero if poll ever decremented load */
/* Configuration */
__u32 update_interval; /* Min load update interval */
__u8 policy; /* Scheduling policy */
/* Synchronization */
pthread_mutex_t cache_lock; /* Lock for structure modifications */
};
/**
* wd_sched_ctx_domain - Scheduling domain with four dimensions
* @region_id: Region identifier (numa_id or device_id)
* @mode: Context mode (SYNC/ASYNC)
* @op_type: Operation type
* @prop: Property (e.g., device type: HW, CE, SOFT)
* @segments: Linked list of context ranges
* @segment_count: Number of segments
* @total_ctx_count: Total contexts across all segments
* @current_segment: Current segment pointer for round-robin
* @current_pos: Current position within segment
* @valid: Domain validity flag
* @lock: Synchronization spinlock
*/
struct wd_sched_ctx_domain {
int region_id;
__u8 mode;
__u32 op_type;
__u8 prop;
struct wd_sched_ctx_segment *segments;
__u32 segment_count;
__u32 total_ctx_count;
struct wd_sched_ctx_segment *current_segment;
__u32 current_pos;
bool valid;
pthread_mutex_t lock;
};
/**
* wd_sched_domain_hash_node - Hash table collision chain node
*/
struct wd_sched_domain_hash_node {
struct wd_sched_ctx_domain domain;
struct wd_sched_domain_hash_node *next;
};
/**
* wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains
* @buckets: Hash table bucket array
* @bucket_size: Number of buckets
* @lock: Read-write lock for concurrent access
*/
struct wd_sched_domain_hash_table {
struct wd_sched_domain_hash_node **buckets;
__u32 bucket_size;
pthread_mutex_t lock;
};
/* ============================================================================
* Dual-Domain Structure for Session Key
* ============================================================================
*/
/**
* wd_sched_key_domain - Session domain with min-heap
* @idx_cache: Index cache with min-heap for load-based selection
* @lock: Synchronization spinlock
* @expanded_count: Track how many times ctx has been expanded
*/
struct wd_sched_key_domain {
struct wd_sched_domain_idx_cache idx_cache;
pthread_mutex_t lock;
__u32 expanded_count;
};
/**
* wd_sched_key - Session-level scheduling key
* @region_id: Region identifier
* @type: Operation type
* @mode: Current mode (SYNC/ASYNC)
* @dev_id: Device identifier (for SCHED_POLICY_DEV)
* @ctx_prop: Context property
* @is_stream: Stream mode flag
* @prio_mode: Priority mode
* @pkt_size: Current packet size
* @sync_domain: Min-heap domain for sync contexts
* @async_domain: Min-heap domain for async contexts
* @lock: Synchronization spinlock
*/
struct wd_sched_key {
int region_id;
__u8 type;
__u8 mode;
__u32 dev_id;
__u8 ctx_prop;
__u16 is_stream;
__u16 prio_mode;
__u32 pkt_size;
struct wd_sched_key_domain sync_domain;
struct wd_sched_key_domain async_domain;
pthread_mutex_t lock;
__u32 poll_lock;
__u32 refcount;
/* Compat filtering parameters for session-ctx matching */
const char *alg_name;
struct wd_ctx_internal *ctxs;
};
/**
* wd_sched_ctx - Main scheduler context
* @policy: Scheduling policy type
* @type_num: Number of operation types
* @mode_num: Number of modes (SYNC/ASYNC)
* @region_num: Number of regions (numa or devices)
* @poll_func: Poll function for receiving responses
* @domain_hash_table: Global hash table for all domains
* @skey_num: Number of active session keys
* @skey_lock: Lock for skey array
* @skey: Array of session keys
*/
struct wd_sched_ctx {
__u32 policy;
__u32 type_num;
__u32 mode_num;
__u16 region_num;
user_poll_func poll_func;
struct wd_sched_domain_hash_table *domain_hash_table;
__u32 skey_num;
pthread_mutex_t skey_lock;
struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM];
/* First ctx index per mode, used by SINGLE/NONE. */
__u32 sync_idx;
__u32 async_idx;
};
/* ============================================================================
* Hash Table Core Operations
* ============================================================================
*/
static bool wd_sched_is_prime(__u32 n)
{
__u32 i;
if (n <= 1)
return false;
if (n <= 3)
return true;
if (n % 2 == 0 || n % 3 == 0)
return false;
for (i = 5; i * i <= n; i += 6) {
if (n % i == 0 || n % (i + 2) == 0)
return false;
}
return true;
}
static __u32 wd_sched_find_prime(__u32 n)
{
while (!wd_sched_is_prime(n))
n++;
return n;
}
static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries)
{
__u32 target_size;
target_size = (estimated_entries * 4) / 3;
if (target_size < WD_SCHED_MIN_BUCKETS)
target_size = WD_SCHED_MIN_BUCKETS;
if (target_size > WD_SCHED_MAX_BUCKETS)
target_size = WD_SCHED_MAX_BUCKETS;
return wd_sched_find_prime(target_size);
}
/**
* wd_sched_hash_compute - Compute hash value for four-dimensional domain key
* @region_id: Region identifier
* @mode: Context mode
* @op_type: Operation type
* @prop: Property
* @bucket_size: Hash table bucket count
*
* Combines four dimensions using prime number multipliers.
*/
static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode,
__u32 op_type, __u8 prop, __u32 bucket_size)
{
__u32 hash;
hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) +
(op_type * HASH_PRIME3) + (prop * HASH_PRIME4);
return hash % bucket_size;
}
static inline bool wd_sched_domain_key_match(
int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1,
int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2)
{
return (region_id1 == region_id2 && mode1 == mode2 &&
op_type1 == op_type2 && prop1 == prop2);
}
/**
* wd_sched_hash_table_create - Create hash table
* @estimated_entries: Estimated number of entries
*
* Returns: Initialized hash table or NULL on error
*/
static struct wd_sched_domain_hash_table *
wd_sched_hash_table_create(__u32 estimated_entries)
{
struct wd_sched_domain_hash_table *table;
__u32 bucket_size;
int ret;
table = calloc(1, sizeof(*table));
if (!table)
return NULL;
bucket_size = wd_sched_compute_bucket_size(estimated_entries);
table->buckets = calloc(bucket_size, sizeof(*table->buckets));
if (!table->buckets) {
free(table);
return NULL;
}
table->bucket_size = bucket_size;
ret = pthread_mutex_init(&table->lock, NULL);
if (ret) {
free(table->buckets);
free(table);
return NULL;
}
return table;
}
static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table)
{
struct wd_sched_domain_hash_node *node, *next;
struct wd_sched_ctx_segment *seg, *next_seg;
__u32 i;
if (!table)
return;
for (i = 0; i < table->bucket_size; i++) {
node = table->buckets[i];
while (node) {
next = node->next;
/* Release segment linked list */
seg = node->domain.segments;
while (seg) {
next_seg = seg->next;
free(seg);
seg = next_seg;
}
pthread_mutex_destroy(&node->domain.lock);
free(node);
node = next;
}
}
pthread_mutex_destroy(&table->lock);
free(table->buckets);
free(table);
}
static struct wd_sched_ctx_domain *
wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table,
int region_id, __u8 mode, __u32 op_type, __u8 prop)
{
struct wd_sched_domain_hash_node *node;
struct wd_sched_ctx_domain *domain = NULL;
__u32 hash_idx;
if (!table)
return NULL;
pthread_mutex_lock(&table->lock);
hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size);
node = table->buckets[hash_idx];
while (node) {
if (wd_sched_domain_key_match(
node->domain.region_id, node->domain.mode, node->domain.op_type,
node->domain.prop, region_id, mode, op_type, prop)) {
domain = &node->domain;
break;
}
node = node->next;
}
pthread_mutex_unlock(&table->lock);
return domain;
}
static struct wd_sched_ctx_domain *
wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table,
int region_id, __u8 mode, __u32 op_type, __u8 prop)
{
struct wd_sched_domain_hash_node *new_node;
struct wd_sched_ctx_domain *existing;
__u32 hash_idx;
int ret;
if (!table)
return NULL;
existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop);
if (existing)
return existing;
pthread_mutex_lock(&table->lock);
hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size);
/* Alloc and initialize new domain */
new_node = calloc(1, sizeof(*new_node));
if (!new_node) {
pthread_mutex_unlock(&table->lock);
return NULL;
}
/* Initialize new domain */
new_node->domain.region_id = region_id;
new_node->domain.mode = mode;
new_node->domain.op_type = op_type;
new_node->domain.prop = prop;
new_node->domain.segments = NULL;
new_node->domain.segment_count = 0;
new_node->domain.total_ctx_count = 0;
new_node->domain.current_segment = NULL;
new_node->domain.current_pos = 0;
new_node->domain.valid = false;
ret = pthread_mutex_init(&new_node->domain.lock, NULL);
if (ret) {
pthread_mutex_unlock(&table->lock);
free(new_node);
return NULL;
}
new_node->next = table->buckets[hash_idx];
table->buckets[hash_idx] = new_node;
pthread_mutex_unlock(&table->lock);
return &new_node->domain;
}
/* ============================================================================
* Segment List Operations
* ============================================================================
*/
/**
* wd_sched_domain_add_segment - Add context range segment to domain
* @domain: Target domain
* @begin: Start context index
* @end: End context index (inclusive)
*
* Supports non-contiguous context ranges via segment list.
*/
static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain,
__u32 begin, __u32 end)
{
struct wd_sched_ctx_segment *seg, *new_seg;
if (!domain || begin > end)
return -WD_EINVAL;
new_seg = calloc(1, sizeof(*new_seg));
if (!new_seg)
return -WD_ENOMEM;
new_seg->begin = begin;
new_seg->end = end;
new_seg->next = NULL;
pthread_mutex_lock(&domain->lock);
/* Append to segment list tail */
if (!domain->segments) {
domain->segments = new_seg;
} else {
seg = domain->segments;
while (seg->next)
seg = seg->next;
seg->next = new_seg;
}
domain->segment_count++;
domain->total_ctx_count += (end - begin + 1);
/* Initialize polling state */
if (!domain->current_segment)
domain->current_segment = domain->segments;
pthread_mutex_unlock(&domain->lock);
return WD_SUCCESS;
}
/**
* wd_sched_domain_get_next_rr - Get next context via round-robin from domain
* @domain: Source domain
*
* Returns: Next global queue index in round-robin order
* Time complexity: O(1)
*/
static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain)
{
__u32 ctx_idx;
__u32 pos;
if (!domain || !domain->segments || !domain->total_ctx_count)
return INVALID_POS;
pthread_mutex_lock(&domain->lock);
if (!domain->current_segment)
domain->current_segment = domain->segments;
pos = domain->current_pos;
/* Calculate global queue number: segment.begin + relative position */
ctx_idx = domain->current_segment->begin + pos;
/* Move to next position */
if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) {
/* Within same segment */
domain->current_pos = pos + 1;
} else if (domain->current_segment->next) {
/* Move to next segment */
domain->current_segment = domain->current_segment->next;
domain->current_pos = 0;
} else {
/* Loop back to beginning */
domain->current_segment = domain->segments;
domain->current_pos = 0;
}
pthread_mutex_unlock(&domain->lock);
return ctx_idx;
}
/* ============================================================================
* SKey Domain Cache Management Functions
* ============================================================================
*/
/**
* wd_sched_skey_cache_init - Initialize skey domain cache
* @cache: Pointer to cache structure
* @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per
* API contract)
*
* Initialize fixed array cache with invalid positions and zero loads.
*/
static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache,
__u8 policy)
{
int i;
if (!cache) {
WD_ERR("invalid: cache pointer is NULL!\n");
return -WD_EINVAL;
}
/* Initialize array with invalid positions */
for (i = 0; i < SKEY_CTX_MAX_NUM; i++) {
cache->idx_list[i] = INVALID_POS;
__atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED);
}
/* Initialize atomic counters */
__atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED);
__atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED);
__atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED);
cache->load_decreased = 0;
/* Set configuration */
cache->valid_count = 0;
cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL;
cache->policy = policy;
/* Initialize structure lock */
if (pthread_mutex_init(&cache->cache_lock, NULL)) {
WD_ERR("failed to init cache lock!\n");
return -WD_EINVAL;
}
return WD_SUCCESS;
}
/**
* wd_sched_skey_cache_uninit - Cleanup skey domain cache
* @cache: Pointer to cache structure
*
* Release resources and reset cache state.
*/
static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache)
{
if (!cache)
return;
pthread_mutex_destroy(&cache->cache_lock);
/* Reset cache state */
for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) {
cache->idx_list[i] = INVALID_POS;
__atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED);
}
cache->valid_count = 0;
cache->load_decreased = 0;
}
/**
* wd_sched_skey_add_ctx - Add ctx to skey domain cache
* @cache: Pointer to cache structure
* @ctx_id: Context ID to add
*
* Add ctx to next available position in fixed array.
* Returns 0 on success, negative error code on failure.
*/
static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache,
__u32 ctx_id)
{
__u32 i;
if (!cache || ctx_id == INVALID_POS) {
WD_ERR("invalid: parameters are NULL!\n");
return -WD_EINVAL;
}
pthread_mutex_lock(&cache->cache_lock);
/* Check if cache is full */
if (cache->valid_count >= SKEY_CTX_MAX_NUM) {
pthread_mutex_unlock(&cache->cache_lock);
WD_ERR("invalid: skey cache full, cannot add more queues!\n");
return -WD_EINVAL;
}
/* Check for duplicate ctx_id */
for (i = 0; i < cache->valid_count; i++) {
if (cache->idx_list[i] == ctx_id) {
WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n",
ctx_id, i);
pthread_mutex_unlock(&cache->cache_lock);
return -WD_EEXIST;
}
}
/* Update min load index if as the new ctx */
__atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED);
/* Add to next available position */
cache->idx_list[cache->valid_count] = ctx_id;
__atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED);
cache->valid_count++;
pthread_mutex_unlock(&cache->cache_lock);
return WD_SUCCESS;
}
/**
* wd_sched_skey_remove_ctx - Remove ctx from skey domain cache
* @cache: Pointer to cache structure
* @ctx_id: Context ID to remove
*
* Remove ctx by shifting array elements to maintain continuity.
* Returns 0 on success, negative error code if not found.
*/
static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache,
__u32 ctx_id)
{
__u32 i, current_min;
int found = 0;
if (!cache) {
WD_ERR("invalid: cache pointer is NULL!\n");
return -WD_EINVAL;
}
pthread_mutex_lock(&cache->cache_lock);
/* Find and remove the ctx */
for (i = 0; i < cache->valid_count; i++) {
if (cache->idx_list[i] == ctx_id) {
found = 1;
break;
}
}
if (!found) {
WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id);
pthread_mutex_unlock(&cache->cache_lock);
return -WD_ENODEV;
}
/* Shift remaining elements to fill the gap */
for (; i < cache->valid_count - 1; i++) {
cache->idx_list[i] = cache->idx_list[i + 1];
__atomic_store_n(&cache->load_values[i],
__atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED),
__ATOMIC_RELAXED);
}
/* Clear last position */
cache->idx_list[cache->valid_count - 1] = INVALID_POS;
__atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED);
cache->valid_count--;
/* Reset pointers if cache becomes empty */
if (!cache->valid_count) {
__atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED);
__atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED);
} else {
/* Adjust min load index if necessary */
current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED);
if (current_min >= cache->valid_count)
__atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED);
}
pthread_mutex_unlock(&cache->cache_lock);
return WD_SUCCESS;
}
/**
* wd_sched_update_min_load - Update cached min load index
* @cache: Pointer to cache structure
* @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values
*
* Scan valid queues to find the one with minimum load.
* When ctxs is provided, reads per-ctx hw_load for cross-session accuracy.
*/
static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache,
struct wd_ctx_internal *ctxs)
{
__u32 min_load = UINT_MAX;
__u32 min_idx = 0;
__u32 i, load;
if (!cache->valid_count)
return;
for (i = 0; i < cache->valid_count; i++) {
if (ctxs)
load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load,
__ATOMIC_RELAXED);
else
load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED);
if (load < min_load) {
min_load = load;
min_idx = i;
}
}
__atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED);
}
/**
* wd_sched_skey_pick_next - Pick next ctx from skey domain cache
* @cache: Pointer to cache structure
* @ctx_idx: Output index within cache array
* @ctxs: Context array for hw_load lookup, or NULL
*
* Select next ctx based on scheduling policy:
* - RR: Simple round-robin selection
* - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available)
*
* Returns selected ctx index, or INVALID_POS if no valid ctx.
*/
static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache,
__u32 *ctx_idx, struct wd_ctx_internal *ctxs)
{
__u32 selected_idx;
__u32 op_count;
if (!cache || !cache->valid_count)
return INVALID_POS;
switch (cache->policy) {
case SCHED_POLICY_RR:
case SCHED_POLICY_NONE:
case SCHED_POLICY_SINGLE:
case SCHED_POLICY_DEV:
case SCHED_POLICY_LOOP:
case SCHED_POLICY_INSTR:
/* Round-robin: atomic increment and module */
selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) %
cache->valid_count;
break;
case SCHED_POLICY_HUNGRY:
/* Update min load periodically */
op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED);
if (op_count % cache->update_interval == 0)
wd_sched_update_min_load(cache, ctxs);
/* Load balancing: use cached min load index */
selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED);
break;
default:
WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy);
selected_idx = INVALID_POS;
break;
}
/* Ensure index is within valid range */
if (selected_idx >= cache->valid_count)
return INVALID_POS;
*ctx_idx = selected_idx;
return cache->idx_list[selected_idx];
}
/**
* wd_sched_skey_update_load - Update load for a specific ctx
* @cache: Pointer to cache structure
* @ctx_idx: Context index in list
* @delta: Load delta (positive for send, negative for receive)
*
* Atomically update load counter for the specified ctx.
* Returns 0 on success, negative error code if ctx not found.
*/
static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache,
__u32 ctx_idx, int delta)
{
/* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */
if (delta > 0)
__atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED);
else {
__atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED);
if (unlikely(!cache->load_decreased))
cache->load_decreased = 1;
}
return WD_SUCCESS;
}
/* ============================================================================
* Session Key Domain Initialization
* ============================================================================
*/
/**
* wd_sched_skey_domain_init - Initialize session domain with min-heap
* @key_domain: Target key domain
* @ctx_idx: context indices idx
* @policy: current session's policy
*
* Initializes dual-domain structure for session.
*/
static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain,
__u32 ctx_idx, __u8 policy)
{
int ret;
if (!key_domain)
return -WD_EINVAL;
ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy);
if (ret)
return ret;
ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx);
if (ret)
goto init_err;
ret = pthread_mutex_init(&key_domain->lock, NULL);
if (ret)
goto add_ctx_err;
key_domain->expanded_count = 0;
return WD_SUCCESS;
add_ctx_err:
wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx);
init_err:
wd_sched_skey_cache_uninit(&key_domain->idx_cache);
return ret;
}
/**
* wd_sched_skey_domain_destroy - Release session domain resources
*/
static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain)
{
if (!key_domain)
return;
pthread_mutex_destroy(&key_domain->lock);
wd_sched_skey_cache_uninit(&key_domain->idx_cache);
}
static __u32 wd_sched_find_compatible_ctx(struct wd_sched_ctx *sched_ctx,
struct wd_sched_key *skey,
int region, int sched_mode)
{
struct wd_sched_ctx_domain *domain;
__u32 i, ctx_idx, prop;
for (prop = 0; prop < UADK_ALG_TYPE_MAX; prop++) {
domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table,
region, sched_mode, skey->type, prop);
if (!domain || !domain->valid)
continue;
for (i = 0; i < domain->total_ctx_count; i++) {
ctx_idx = wd_sched_domain_get_next_rr(domain);
if (ctx_idx == INVALID_POS)
continue;
if (skey->ctxs[ctx_idx].drv &&
wd_alg_match_drv(skey->ctxs[ctx_idx].drv, skey->alg_name))
return ctx_idx;
}
}
return INVALID_POS;
}
static __u32 wd_sched_get_new_ctx(struct wd_sched_ctx *sched_ctx,
struct wd_sched_key *skey,
int sched_mode)
{
int region_id = skey->region_id;
__u8 ctx_prop = skey->ctx_prop;
__u32 op_type = skey->type;
__u32 ctx_idx;
int r;
if (sched_mode >= SCHED_MODE_BUTT ||
op_type >= sched_ctx->type_num || ctx_prop >= UADK_ALG_TYPE_MAX) {
WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n",
region_id, sched_mode, op_type, ctx_prop);
return INVALID_POS;
}
if (region_id < 0 ||
(sched_ctx->policy != SCHED_POLICY_DEV &&
region_id >= sched_ctx->region_num)) {
WD_ERR("invalid: region_id is %d, region_num is %u!\n",
region_id, sched_ctx->region_num);
return INVALID_POS;
}
if (!sched_ctx->domain_hash_table)
return INVALID_POS;
/* Try current region first */
ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, region_id, sched_mode);
if (ctx_idx != INVALID_POS)
return ctx_idx;
/* DEV policy must not cross region */
if (sched_ctx->policy == SCHED_POLICY_DEV)
return INVALID_POS;
/* Cross-region fallback: try all other regions */
for (r = 0; r < sched_ctx->region_num; r++) {
if (r == region_id)
continue;
ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, r, sched_mode);
if (ctx_idx != INVALID_POS)
return ctx_idx;
}
return INVALID_POS;
}
/**
* wd_sched_skey_compat_filter - Filter and replace incompatible ctxs in domain cache
* @sched_ctx: Scheduler context
* @skey: Session key with alg_name and ctxs
* @domain: Target domain (sync or async)
* @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC
*
* For each ctx in domain cache, check if it supports alg_name.
* If not, find a compatible replacement from the global domain.
*/
static void wd_sched_skey_compat_filter(struct wd_sched_ctx *sched_ctx,
struct wd_sched_key *skey, struct wd_sched_key_domain *domain, int sched_mode)
{
__u32 ctx_idx, new_ctx;
__u32 i;
if (!skey || !skey->alg_name || !skey->ctxs || !domain)
return;
/* Skip uninitialized domains (no ctxs cached) */