diff --git a/sys/cpu/x86_64/include/atomic.h b/sys/cpu/x86_64/include/atomic.h index 1f6c0d5a9e..9cf304233d 100644 --- a/sys/cpu/x86_64/include/atomic.h +++ b/sys/cpu/x86_64/include/atomic.h @@ -382,11 +382,11 @@ atomic_intr_cond_exit(__atomic_intr_t *p, void (*func)(void *), void *arg) static __inline int atomic_cmpxchg_int(volatile u_int *_dst, u_int _old, u_int _new) { - u_int res = _old; + u_int res; __asm __volatile(MPLOCKED "cmpxchgl %2,%1; " \ - : "+a" (res), "=m" (*_dst) \ - : "r" (_new), "m" (*_dst) \ + : "=a" (res), "+m" (*_dst) \ + : "r" (_new), "0" (_old) \ : "memory"); return (res); } @@ -683,7 +683,7 @@ atomic_testandclear_long(volatile u_long *p, u_long v) return (res); } -#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ +#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP, MOP) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ @@ -698,6 +698,28 @@ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ return (res); \ } \ \ +/* \ + * LOCK cannot be used with MOV, but XCHG cannot be \ + * if implementing a write to memory-mapped I/O. Use \ + * a regular memmory move with a compiler barrier \ + * and MFENCE before and after \ + */ \ +static __inline u_##TYPE \ +atomic_read_##TYPE(volatile u_##TYPE *p) \ +{ \ + u_##TYPE res; /* accumulator can be anything */ \ + \ + __asm __volatile("mfence" : : : "memory"); \ + __asm __volatile(MOP \ + : "=a" (res), /* 0 */ \ + "=m" (*p) /* 1 */ \ + : "m" (*p) /* 2 */ \ + : "memory"); \ + __asm __volatile("mfence" : : : "memory"); \ + \ + return (res); \ +} \ + \ /* \ * The XCHG instruction asserts LOCK automagically. \ */ \ @@ -707,14 +729,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ __asm __volatile(SOP \ : "=m" (*p), /* 0 */ \ "+r" (v) /* 1 */ \ - : "m" (*p)); /* 2 */ \ + : "m" (*p) /* 2 */ \ + : "memory"); \ +} \ + \ +/* \ + * LOCK cannot be used with MOV, but XCHG cannot be \ + * if implementing a write to memory-mapped I/O. Use \ + * a regular memmory move with a compiler barrier \ + * and MFENCE before and after \ + */ \ + \ +static __inline void \ +atomic_write_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + __asm __volatile("mfence" : : : "memory"); \ + __asm __volatile(MOP \ + : "=m" (*p), /* 0 */ \ + "+r" (v) /* 1 */ \ + : "m" (*p) /* 2 */ \ + : "memory"); \ + __asm __volatile("mfence" : : : "memory"); \ } \ struct __hack -ATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0"); -ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0"); -ATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0"); -ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); +ATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0", "movb %b1,%0"); +ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0", "movw %w1,%0"); +ATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0", "movl %1,%0"); +ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0", "movq %1,%0"); #undef ATOMIC_ASM #undef ATOMIC_STORE_LOAD @@ -768,6 +810,8 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #define atomic_load_acq_cpumask atomic_load_acq_long /* Operations on 8-bit bytes. */ +#define atomic_read_8 atomic_read_char +#define atomic_write_8 atomic_write_char #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char @@ -785,6 +829,8 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #define atomic_fcmpset_8 atomic_fcmpset_char /* Operations on 16-bit words. */ +#define atomic_read_16 atomic_read_short +#define atomic_write_16 atomic_write_short #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short @@ -802,6 +848,8 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #define atomic_fcmpset_16 atomic_fcmpset_short /* Operations on 32-bit double words. */ +#define atomic_read_32 atomic_read_int +#define atomic_write_32 atomic_write_int #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int @@ -824,6 +872,8 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #define atomic_fetchadd_32 atomic_fetchadd_int /* Operations on 64-bit quad words. */ +#define atomic_read_64 atomic_read_long +#define atomic_write_64 atomic_write_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_swap_64 atomic_swap_long @@ -863,6 +913,8 @@ ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); atomic_load_acq_long((volatile u_long *)(p)) #define atomic_store_rel_ptr(p, v) \ atomic_store_rel_long((volatile u_long *)(p), (v)) +#define atomic_write_ptr(p, v) \ + atomic_write_long((volatile u_long *)(p), (v)) #define atomic_cmpset_ptr(dst, old, new) \ atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), \ (u_long)(new)) diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_cs.c b/sys/dev/drm/amd/amdgpu/amdgpu_cs.c index 513be9cf3d..ba63c50a51 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_cs.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_cs.c @@ -34,6 +34,37 @@ #include "amdgpu_gmc.h" #include "amdgpu_gem.h" +#if 0 +static void amdgpu_cs_check_duplicates(struct amdgpu_cs_parser *p, + const char *str) +{ + struct amdgpu_bo_list_entry *e1; + struct amdgpu_bo_list_entry *e2; + int i; + int j; + + if (list_empty(&p->validated)) + return; + + i = 0; + list_for_each_entry(e1, &p->validated, tv.head) { + struct amdgpu_bo *b1 = ttm_to_amdgpu_bo(e1->tv.bo); + j = 0; + list_for_each_entry(e2, &p->validated, tv.head) { + struct amdgpu_bo *b2 = ttm_to_amdgpu_bo(e2->tv.bo); + + if (e1 == e2) + break; + if (b1 == b2) { + kprintf("conflicting entries %s amdgpu_bo=%p %d/%d\n", str, b1, j, i); + } + ++j; + } + ++i; + } +} +#endif + static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, struct drm_amdgpu_cs_chunk_fence *data, uint32_t *offset) @@ -601,6 +632,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, } amdgpu_bo_list_get_list(p->bo_list, &p->validated); + //amdgpu_cs_check_duplicates(p, "J"); if (p->bo_list->first_userptr != p->bo_list->num_entries) p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX); @@ -790,9 +822,10 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, { unsigned i; - if (error && backoff) + if (error && backoff) { ttm_eu_backoff_reservation(&parser->ticket, &parser->validated); + } for (i = 0; i < parser->num_post_dep_syncobjs; i++) drm_syncobj_put(parser->post_dep_syncobjs[i]); diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_ctx.c b/sys/dev/drm/amd/amdgpu/amdgpu_ctx.c index a9b3927d3b..ad32958ca8 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_ctx.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_ctx.c @@ -126,6 +126,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; struct drm_sched_rq *rqs[AMDGPU_MAX_RINGS]; unsigned num_rings; + unsigned num_rqs = 0; switch (i) { case AMDGPU_HW_IP_GFX: @@ -168,12 +169,16 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev, break; } - for (j = 0; j < num_rings; ++j) - rqs[j] = &rings[j]->sched.sched_rq[priority]; + for (j = 0; j < num_rings; ++j) { + if (!rings[j]->adev) + continue; + + rqs[num_rqs++] = &rings[j]->sched.sched_rq[priority]; + } for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) r = drm_sched_entity_init(&ctx->entities[i][j].entity, - rqs, num_rings, &ctx->guilty); + rqs, num_rqs, &ctx->guilty); if (r) goto error_cleanup_entities; } @@ -442,18 +447,33 @@ void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct dma_fence *fence, uint64_t* handle) { struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); - uint64_t seq = centity->sequence; + uint64_t seq; struct dma_fence *other = NULL; unsigned idx = 0; +retry: + lockmgr(&ctx->ring_lock, LK_EXCLUSIVE); + seq = centity->sequence; idx = seq & (amdgpu_sched_jobs - 1); other = centity->fences[idx]; - if (other) - BUG_ON(!dma_fence_is_signaled(other)); + if (other) { + if (!dma_fence_is_signaled(other)) { + lockmgr(&ctx->ring_lock, LK_RELEASE); + + kprintf("amdgpu_ctx_add_fence: incomplete fence " + "pid=%d tid=%d seq=%ld idx=%d centity=%p other=%p\n", + (curproc ? curproc->p_pid : -1), + (curthread->td_lwp ? curthread->td_lwp->lwp_tid : -1), + seq, idx, centity, other); + for (;;) + tsleep(&seq, 0, "fenceslp", 0); /* just stop */ + goto retry; + } + //BUG_ON(!dma_fence_is_signaled(other)); + } dma_fence_get(fence); - lockmgr(&ctx->ring_lock, LK_EXCLUSIVE); centity->fences[idx] = fence; centity->sequence++; lockmgr(&ctx->ring_lock, LK_RELEASE); diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_device.c b/sys/dev/drm/amd/amdgpu/amdgpu_device.c index e43fd97512..8023180e52 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_device.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_device.c @@ -3372,6 +3372,7 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) DRM_INFO("GPU recovery disabled.\n"); return false; } + kprintf("GPU recovery attempt\n"); return true; } diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_drv.c b/sys/dev/drm/amd/amdgpu/amdgpu_drv.c index 043fe5d7c6..7de039ef84 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_drv.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_drv.c @@ -105,7 +105,7 @@ int amdgpu_vram_page_split = 512; int amdgpu_vm_update_mode = -1; int amdgpu_exp_hw_support = 0; int amdgpu_dc = -1; -int amdgpu_sched_jobs = 32; +int amdgpu_sched_jobs = 128; int amdgpu_sched_hw_submission = 2; uint amdgpu_pcie_gen_cap = 0; uint amdgpu_pcie_lane_cap = 0; @@ -125,6 +125,7 @@ int amdgpu_job_hang_limit = 0; int amdgpu_lbpw = -1; int amdgpu_compute_multipipe = -1; int amdgpu_gpu_recovery = -1; /* auto */ +//int amdgpu_gpu_recovery = 1; /* try it */ int amdgpu_emu_mode = 0; uint amdgpu_smu_memory_pool_size = 0; /* FBC (bit 0) disabled by default*/ diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_gem.c b/sys/dev/drm/amd/amdgpu/amdgpu_gem.c index 1d8f458dcf..4fb3478317 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_gem.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_gem.c @@ -54,10 +54,13 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, memset(&bp, 0, sizeof(bp)); *obj = NULL; +#if 0 + /* ZZZ remove */ /* At least align on page size */ if (alignment < PAGE_SIZE) { alignment = PAGE_SIZE; } +#endif bp.size = size; bp.byte_align = alignment; @@ -244,9 +247,12 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, return -EINVAL; } flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS; +#if 0 + /* ZZZ remove */ /* GDS allocations must be DW aligned */ if (args->in.domains & AMDGPU_GEM_DOMAIN_GDS) size = ALIGN(size, 4); +#endif } if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_object.c b/sys/dev/drm/amd/amdgpu/amdgpu_object.c index dda3df2420..20674df467 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_object.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_object.c @@ -431,12 +431,51 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, size_t acc_size; int r; +#if 1 page_align = roundup(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; if (bp->domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) size <<= PAGE_SHIFT; else size = ALIGN(size, PAGE_SIZE); +#endif + +#if 0 /* ZZZ linux patch, not consistent for GSD and GWS/OA vs DEFAULT */ + /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ + if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { + /* GWS and OA don't need any alignment. */ + page_align = bp->byte_align; + size <<= PAGE_SHIFT; + kprintf("GWS/OA: page_align %ld size %ld\n", page_align, size); + } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { + /* Both size and alignment must be a multiple of 4. */ + page_align = ALIGN(bp->byte_align, 4); + size = ALIGN(size, 4) << PAGE_SHIFT; + kprintf(" GDS: page_align %ld size %ld\n", page_align, size); + } else { + /* Memory should be aligned at least to a page size. */ + page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; + size = ALIGN(size, PAGE_SIZE); + kprintf("DEFALT: page_align %ld size %ld\n", page_align, size); + } +#endif + +#if 1 + /* ZZZ force alignment to fit the allocation size */ + if (size > 2048 * 1024) { + unsigned long opa = page_align; + if (page_align == 0) + page_align = 1; + while (page_align * PAGE_SIZE < size) { + page_align <<= 1; + kprintf("ZZZ increase page_align from %ld to %ld\n", + opa, page_align); + } + } +#endif + + kprintf("amdgpu_bo_create: size=%ld pga=%ld (ba=%d)\n", /* ZZZ */ + size, page_align, bp->byte_align); if (!amdgpu_bo_validate_size(adev, size, bp->domain)) return -ENOMEM; diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_ttm.c b/sys/dev/drm/amd/amdgpu/amdgpu_ttm.c index 22b2c18c89..5f5449ce32 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_ttm.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_ttm.c @@ -188,14 +188,22 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_MASK_CACHING; man->default_caching = TTM_PL_FLAG_CACHED; + + // YYY OVERRIDE + //man->available_caching = TTM_PL_MASK_CACHING; /* YYY */ + //man->default_caching = TTM_PL_FLAG_WC; /* YYY */ break; case TTM_PL_TT: /* GTT memory */ man->func = &amdgpu_gtt_mgr_func; man->gpu_offset = adev->gmc.gart_start; man->available_caching = TTM_PL_MASK_CACHING; - man->default_caching = TTM_PL_FLAG_CACHED; + man->default_caching = TTM_PL_FLAG_CACHED; //TTM_PL_FLAG_UNCACHED; //TTM_PL_FLAG_CACHED; man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; + + // YYY OVERRIDE + //man->available_caching = TTM_PL_MASK_CACHING; /* YYY */ + //man->default_caching = TTM_PL_FLAG_WC; /* YYY */ break; case TTM_PL_VRAM: /* "On-card" video ram */ @@ -204,7 +212,9 @@ static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_MAPPABLE; man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; - man->default_caching = TTM_PL_FLAG_WC; + man->default_caching = TTM_PL_FLAG_WC; //TTM_PL_FLAG_WC; + // YYY OVERRIDE + //man->default_caching = TTM_PL_FLAG_WC; break; case AMDGPU_PL_GDS: case AMDGPU_PL_GWS: @@ -321,8 +331,13 @@ static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) if (amdgpu_ttm_tt_get_usermm(bo->ttm)) return -EPERM; + +#ifdef __DragonFly__ + return 0; +#else return drm_vma_node_verify_access(&abo->gem_base.vma_node, filp->private_data); +#endif } /** @@ -428,9 +443,10 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, mutex_lock(&adev->mman.gtt_window_lock); + kprintf("mem_to_mem: %ld bytes %p %p\n", size, src_mm, dst_mm); while (size) { unsigned long cur_size; - uint64_t from = src_node_start, to = dst_node_start; + uint64_t from, to; struct dma_fence *next; /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst @@ -442,31 +458,36 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, cur_size + dst_page_offset > GTT_MAX_BYTES) cur_size -= max(src_page_offset, dst_page_offset); + kprintf(" loop: %ld bytes %p %p - cur=%ld " + "src(mem=%016lx,node=%lu/%lu,of=%ld) dst(mem=%016lx,node=%lu/%lu,of=%ld)\n", + size, src_mm, dst_mm, cur_size, + src->mem->start, src_node_start, src_node_size, src_page_offset, + dst->mem->start, dst_node_start, dst_node_size, dst_page_offset); + /* Map only what needs to be accessed. Map src to window 0 and * dst to window 1 */ - if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) { - r = amdgpu_map_buffer(src->bo, src->mem, - PFN_UP(cur_size + src_page_offset), - src_node_start, 0, ring, - &from); - if (r) - goto error; - /* Adjust the offset because amdgpu_map_buffer returns - * start of mapped page - */ - from += src_page_offset; - } + r = amdgpu_map_buffer(src->bo, src->mem, + PFN_UP(cur_size + src_page_offset), + src_node_start, 0, ring, + &from); + if (r) + goto error; + /* Adjust the offset because amdgpu_map_buffer returns + * start of mapped page + */ + from += src_page_offset; - if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) { - r = amdgpu_map_buffer(dst->bo, dst->mem, - PFN_UP(cur_size + dst_page_offset), - dst_node_start, 1, ring, - &to); - if (r) - goto error; - to += dst_page_offset; - } + r = amdgpu_map_buffer(dst->bo, dst->mem, + PFN_UP(cur_size + dst_page_offset), + dst_node_start, 1, ring, + &to); + if (r) + goto error; + to += dst_page_offset; + + kprintf(" ext: from=%lu to=%lu cur_size=%ld\n", + from, to, cur_size); r = amdgpu_copy_buffer(ring, from, to, cur_size, resv, &next, false, true); @@ -474,6 +495,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, goto error; dma_fence_put(fence); + //dma_fence_wait(next, false); /* ZZZ */ fence = next; size -= cur_size; @@ -482,18 +504,26 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, src_node_size -= cur_size; if (!src_node_size) { - src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm, + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + ++src_mm; + src_node_size = src_mm->size << PAGE_SHIFT; + src_page_offset = 0; + src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem); - src_node_size = (src_mm->size << PAGE_SHIFT); + src->offset = 0; } else { src_node_start += cur_size; src_page_offset = src_node_start & (PAGE_SIZE - 1); } dst_node_size -= cur_size; if (!dst_node_size) { - dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm, - dst->mem); + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + ++dst_mm; dst_node_size = (dst_mm->size << PAGE_SHIFT); + dst_page_offset = 0; + dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, + dst->mem); + dst->offset = 0; } else { dst_node_start += cur_size; dst_page_offset = dst_node_start & (PAGE_SIZE - 1); @@ -836,6 +866,9 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) unsigned pinned = 0; int r; + kprintf("amdgpu_ttm_tt_get_user_pages caching_state: %d\n", + ttm->caching_state); + if (!mm) /* Happens during process shutdown */ return -ESRCH; @@ -2029,6 +2062,23 @@ static int amdgpu_map_buffer(struct ttm_buffer_object *bo, BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); + /* ZZZ linux */ +#if 1 + /* Map only what can't be accessed directly */ + if (mem->start != AMDGPU_BO_INVALID_OFFSET) { + kprintf("x"); + *addr = offset; + return 0; + } +#endif + /* ZZZ dillon early use */ + if (gtt == NULL) { + kprintf("AMDGPU_NOMAP\n"); + *addr = offset; + return 0; + } + kprintf("AMDGPU_MAP_BUFFER\n"); + *addr = adev->gmc.gart_start; *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GPU_PAGE_SIZE; @@ -2061,6 +2111,11 @@ static int amdgpu_map_buffer(struct ttm_buffer_object *bo, if (r) goto error_free; + /* ZZZ */ + //mb(); + //amdgpu_asic_flush_hdp(adev, NULL); + //amdgpu_gmc_flush_gpu_tlb(adev, 0); + r = amdgpu_job_submit(job, &adev->mman.entity, AMDGPU_FENCE_OWNER_UNDEFINED, &fence); if (r) @@ -2180,9 +2235,11 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, mm_node = bo->tbo.mem.mm_node; num_loops = 0; while (num_pages) { - uint32_t byte_count = mm_node->size << PAGE_SHIFT; + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + uint64_t byte_count = mm_node->size << PAGE_SHIFT; - num_loops += DIV_ROUND_UP(byte_count, max_bytes); + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes); num_pages -= mm_node->size; ++mm_node; } @@ -2208,12 +2265,15 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, mm_node = bo->tbo.mem.mm_node; while (num_pages) { - uint32_t byte_count = mm_node->size << PAGE_SHIFT; + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + uint64_t byte_count = mm_node->size << PAGE_SHIFT; uint64_t dst_addr; dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem); while (byte_count) { - uint32_t cur_size_in_bytes = min(byte_count, max_bytes); + /* linux:9288b874c5006a22d1fbff303b812cb03241cdfb */ + uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count, + max_bytes); amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, cur_size_in_bytes); @@ -2376,7 +2436,7 @@ static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf, while (size) { loff_t p = *pos / PAGE_SIZE; - unsigned off = *pos & ~PAGE_MASK; + unsigned off = *pos & ~LINUX_PAGE_MASK; size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); struct page *page; void *ptr; @@ -2433,8 +2493,8 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, dom = iommu_get_domain_for_dev(adev->dev); while (size) { - phys_addr_t addr = *pos & PAGE_MASK; - loff_t off = *pos & ~PAGE_MASK; + phys_addr_t addr = *pos & LINUX_PAGE_MASK; + loff_t off = *pos & ~LINUX_PAGE_MASK; size_t bytes = PAGE_SIZE - off; unsigned long pfn; struct page *p; @@ -2490,8 +2550,8 @@ static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, dom = iommu_get_domain_for_dev(adev->dev); while (size) { - phys_addr_t addr = *pos & PAGE_MASK; - loff_t off = *pos & ~PAGE_MASK; + phys_addr_t addr = *pos & LINUX_PAGE_MASK; + loff_t off = *pos & ~LINUX_PAGE_MASK; size_t bytes = PAGE_SIZE - off; unsigned long pfn; struct page *p; diff --git a/sys/dev/drm/amd/amdgpu/amdgpu_vm.c b/sys/dev/drm/amd/amdgpu/amdgpu_vm.c index fde703c982..e90b186cf7 100644 --- a/sys/dev/drm/amd/amdgpu/amdgpu_vm.c +++ b/sys/dev/drm/amd/amdgpu/amdgpu_vm.c @@ -1338,7 +1338,7 @@ static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) result = pages_addr[addr >> PAGE_SHIFT]; /* in case cpu page size != gpu page size*/ - result |= addr & (~PAGE_MASK); + result |= addr & (~LINUX_PAGE_MASK); result &= 0xFFFFFFFFFFFFF000ULL; @@ -3313,7 +3313,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { dev_err(adev->dev, "still active bo inside vm\n"); } -#ifndef __DragonFly__ +#if 1 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va.rb_root, rb) { #else @@ -3513,7 +3513,6 @@ void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, */ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) { - kprintf("amdgpu_vm_set_task_info: not implemented\n"); #if 0 if (!vm->task_info.pid) { vm->task_info.pid = current->pid; diff --git a/sys/dev/drm/amd/amdgpu/gfx_v8_0.c b/sys/dev/drm/amd/amdgpu/gfx_v8_0.c index 443693083f..22556be161 100644 --- a/sys/dev/drm/amd/amdgpu/gfx_v8_0.c +++ b/sys/dev/drm/amd/amdgpu/gfx_v8_0.c @@ -6184,7 +6184,23 @@ static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, uint64_t addr bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; - /* EVENT_WRITE_EOP - flush caches, send int */ + /* Workaround for cache flush problems. First send a dummy EOP + * event down the pipe with seq one below. + */ + amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); + amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | + EOP_TC_ACTION_EN | + EOP_TC_WB_ACTION_EN | + EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | + EVENT_INDEX(5))); + amdgpu_ring_write(ring, addr & 0xfffffffc); + amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | + DATA_SEL(1) | INT_SEL(0)); + amdgpu_ring_write(ring, lower_32_bits(seq - 1)); + amdgpu_ring_write(ring, upper_32_bits(seq - 1)); + + /* Then send the real EOP event down the pipe: + * EVENT_WRITE_EOP - flush caches, send int */ amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN | @@ -6899,7 +6915,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 5 + /* COND_EXEC */ 7 + /* PIPELINE_SYNC */ VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */ - 8 + /* FENCE for VM_FLUSH */ + 12 + /* FENCE for VM_FLUSH */ 20 + /* GDS switch */ 4 + /* double SWITCH_BUFFER, the first COND_EXEC jump to the place just @@ -6911,7 +6927,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 31 + /* DE_META */ 3 + /* CNTX_CTRL */ 5 + /* HDP_INVL */ - 8 + 8 + /* FENCE x2 */ + 12 + 12 + /* FENCE x2 */ 2, /* SWITCH_BUFFER */ .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */ .emit_ib = gfx_v8_0_ring_emit_ib_gfx, diff --git a/sys/dev/drm/amd/amdgpu/gmc_v7_0.c b/sys/dev/drm/amd/amdgpu/gmc_v7_0.c index 98b7d81c53..241c3eb5fc 100644 --- a/sys/dev/drm/amd/amdgpu/gmc_v7_0.c +++ b/sys/dev/drm/amd/amdgpu/gmc_v7_0.c @@ -718,6 +718,7 @@ static int gmc_v7_0_gart_init(struct amdgpu_device *adev) r = amdgpu_gart_init(adev); if (r) return r; + kprintf("gmc_v7_0_gart_init\n"); adev->gart.table_size = adev->gart.num_gpu_pages * 8; adev->gart.gart_pte_flags = 0; return amdgpu_gart_table_vram_alloc(adev); diff --git a/sys/dev/drm/amd/amdgpu/gmc_v8_0.c b/sys/dev/drm/amd/amdgpu/gmc_v8_0.c index e0d0f5be30..2a18782268 100644 --- a/sys/dev/drm/amd/amdgpu/gmc_v8_0.c +++ b/sys/dev/drm/amd/amdgpu/gmc_v8_0.c @@ -962,6 +962,7 @@ static int gmc_v8_0_gart_init(struct amdgpu_device *adev) r = amdgpu_gart_init(adev); if (r) return r; + kprintf("gmc_v8_0_gart_init\n"); adev->gart.table_size = adev->gart.num_gpu_pages * 8; adev->gart.gart_pte_flags = AMDGPU_PTE_EXECUTABLE; return amdgpu_gart_table_vram_alloc(adev); diff --git a/sys/dev/drm/amd/amdgpu/gmc_v9_0.c b/sys/dev/drm/amd/amdgpu/gmc_v9_0.c index d7f10c4b72..87be305abc 100644 --- a/sys/dev/drm/amd/amdgpu/gmc_v9_0.c +++ b/sys/dev/drm/amd/amdgpu/gmc_v9_0.c @@ -517,6 +517,8 @@ static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, value = addr & 0x0000FFFFFFFFF000ULL; value |= flags; writeq(value, ptr + (gpu_page_idx * 8)); + //mb(); /* ZZZ */ + //gmc_v9_0_flush_gpu_tlb(adev, 0); /* ZZZ */ return 0; } @@ -871,6 +873,7 @@ static int gmc_v9_0_gart_init(struct amdgpu_device *adev) r = amdgpu_gart_init(adev); if (r) return r; + kprintf("gmc_v9_0_gart_init\n"); adev->gart.table_size = adev->gart.num_gpu_pages * 8; adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | AMDGPU_PTE_EXECUTABLE; diff --git a/sys/dev/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/sys/dev/drm/amd/display/amdgpu_dm/amdgpu_dm.c index f591f7119f..0e2453c1d5 100644 --- a/sys/dev/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/sys/dev/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1616,8 +1616,15 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev) &dm_atomic_state_funcs); r = amdgpu_display_modeset_create_props(adev); - if (r) + if (r) { + /* + * YYY memory leak fix from + * e035803797473c90052244dcf8eaec455f0c884f (but nothing else) + */ + dc_release_state(state->context); + kfree(state); return r; + } return 0; } diff --git a/sys/dev/drm/drm_cache.c b/sys/dev/drm/drm_cache.c index c56adaf020..9f30c2fa71 100644 --- a/sys/dev/drm/drm_cache.c +++ b/sys/dev/drm/drm_cache.c @@ -40,21 +40,15 @@ * clflushopt is an unordered instruction which needs fencing with mfence or * sfence to avoid ordering issues. For drm_clflush_page this fencing happens * in the caller. + * + * The page may represent PCIe mapped memory or real physical memory. Note + * that the DMAP only works with real physical memory. */ static void drm_clflush_page(struct page *page) { - uint8_t *page_virtual; - unsigned int i; - const int size = boot_cpu_data.x86_clflush_size; - - if (unlikely(page == NULL)) - return; - - page_virtual = kmap_atomic(page); - for (i = 0; i < PAGE_SIZE; i += size) - clflushopt(page_virtual + i); - kunmap_atomic(page_virtual); + if (page) + pmap_invalidate_cache_page((vm_page_t)page); } static void drm_cache_flush_clflush(struct page *pages[], @@ -64,7 +58,7 @@ static void drm_cache_flush_clflush(struct page *pages[], mb(); for (i = 0; i < num_pages; i++) - drm_clflush_page(*pages++); + drm_clflush_page(pages[i]); mb(); } #endif diff --git a/sys/dev/drm/drm_dp_helper.c b/sys/dev/drm/drm_dp_helper.c index 127be3ecf0..046137eadf 100644 --- a/sys/dev/drm/drm_dp_helper.c +++ b/sys/dev/drm/drm_dp_helper.c @@ -236,7 +236,7 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, * aux i2c transactions but real world devices this wasn't * sufficient, bump to 32 which makes Dell 4k monitors happier. */ - for (retry = 0; retry < 32; retry++) { + for (retry = 0; retry < 320; retry++) { if (ret != 0 && ret != -ETIMEDOUT) { usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100); diff --git a/sys/dev/drm/drm_modeset_lock.c b/sys/dev/drm/drm_modeset_lock.c index 0351d0f9d0..fea6f0142f 100644 --- a/sys/dev/drm/drm_modeset_lock.c +++ b/sys/dev/drm/drm_modeset_lock.c @@ -271,7 +271,9 @@ static inline int modeset_lock(struct drm_modeset_lock *lock, * we will need to be able to drm_modeset_lock() things * without having to keep track of what is already locked * or not. + * XXX but ww_mutex isn't recursive. bad hack here for now */ + ww_mutex_lock_recursive(&lock->mutex); ret = 0; } else if (ret == -EDEADLK) { ctx->contended = lock; diff --git a/sys/dev/drm/drm_sysctl.c b/sys/dev/drm/drm_sysctl.c index d40ca6c60b..7df4ce3db4 100644 --- a/sys/dev/drm/drm_sysctl.c +++ b/sys/dev/drm/drm_sysctl.c @@ -49,6 +49,7 @@ static int drm_name_info DRM_SYSCTL_HANDLER_ARGS; static int drm_vm_info DRM_SYSCTL_HANDLER_ARGS; static int drm_clients_info DRM_SYSCTL_HANDLER_ARGS; static int drm_bufs_info DRM_SYSCTL_HANDLER_ARGS; +static int drm_busid_info DRM_SYSCTL_HANDLER_ARGS; struct drm_sysctl_list { const char *name; @@ -58,6 +59,7 @@ struct drm_sysctl_list { {"vm", drm_vm_info}, {"clients", drm_clients_info}, {"bufs", drm_bufs_info}, + {"busid", drm_busid_info}, }; #define DRM_SYSCTL_ENTRIES NELEM(drm_sysctl_list) @@ -298,3 +300,25 @@ static int drm_clients_info DRM_SYSCTL_HANDLER_ARGS kfree(tempprivs); return retcode; } + +static int drm_busid_info DRM_SYSCTL_HANDLER_ARGS +{ + struct drm_device *dev = arg1; + char buf[128]; + device_t bsddev; + int domain, bus, slot, func; + int retcode = 0; + + bsddev = dev->dev->bsddev; + domain = pci_get_domain(bsddev); + bus = pci_get_bus(bsddev); + slot = pci_get_slot(bsddev); + func = pci_get_function(bsddev); + + DRM_SYSCTL_PRINT("pci:%04x:%02x:%02x.%d", domain, bus, slot, func); + + SYSCTL_OUT(req, "", 1); + +done: + return retcode; +} diff --git a/sys/dev/drm/i915/i915_gem.c b/sys/dev/drm/i915/i915_gem.c index 1e826e0be5..8760e4d181 100644 --- a/sys/dev/drm/i915/i915_gem.c +++ b/sys/dev/drm/i915/i915_gem.c @@ -318,6 +318,7 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) } src = kmap_atomic(page); + drm_clflush_virt_range(src, PAGE_SIZE); memcpy(vaddr, src, PAGE_SIZE); drm_clflush_virt_range(vaddr, PAGE_SIZE); kunmap_atomic(src); @@ -410,8 +411,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, continue; dst = kmap_atomic(page); - drm_clflush_virt_range(vaddr, PAGE_SIZE); memcpy(dst, vaddr, PAGE_SIZE); + drm_clflush_virt_range(vaddr, PAGE_SIZE); kunmap_atomic(dst); set_page_dirty(page); diff --git a/sys/dev/drm/include/asm/bug.h b/sys/dev/drm/include/asm/bug.h index c6852ae4e3..3fe9be7298 100644 --- a/sys/dev/drm/include/asm/bug.h +++ b/sys/dev/drm/include/asm/bug.h @@ -75,6 +75,7 @@ if ((condition) && !__warned_once) { \ WARN(condition, format); \ __warned_once = true; \ + print_backtrace(-1); \ } \ unlikely(__ret); \ }) diff --git a/sys/dev/drm/include/asm/cacheflush.h b/sys/dev/drm/include/asm/cacheflush.h index a21ddee334..f486858937 100644 --- a/sys/dev/drm/include/asm/cacheflush.h +++ b/sys/dev/drm/include/asm/cacheflush.h @@ -35,38 +35,74 @@ static inline int set_memory_uc(unsigned long addr, int numpages) { - pmap_change_attr(addr, numpages, PAT_UNCACHED); +kprintf("set_memory_uc\n"); + pmap_change_attr(addr, numpages, PAT_UNCACHEABLE); return 0; } static inline int set_memory_wc(unsigned long vaddr, int numpages) { +kprintf("set_memory_wc\n"); pmap_change_attr(vaddr, numpages, PAT_WRITE_COMBINING); + //pmap_change_attr(vaddr, numpages, PAT_UNCACHEABLE); /* YYY */ return 0; } static inline int set_memory_wb(unsigned long vaddr, int numpages) { +kprintf("set_memory_wb\n"); pmap_change_attr(vaddr, numpages, PAT_WRITE_BACK); + //pmap_change_attr(vaddr, numpages, PAT_UNCACHEABLE); /* YYY */ return 0; } static inline int set_pages_uc(struct page *page, int num_pages) { - struct vm_page *p = (struct vm_page *)page; - - pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), - num_pages, PAT_UNCACHED); + kprintf("set_pages_uc\n"); + //pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), + // num_pages, PAT_UNCACHEABLE); + for (int i = 0; i < num_pages; i++) { + vm_page_t p = (vm_page_t)&page[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_UNCACHEABLE); + pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); + } return 0; } static inline int set_pages_wb(struct page *page, int num_pages) { - struct vm_page *p = (struct vm_page *)page; + //kprintf("set_pages_wb\n"); + //pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), + // num_pages, PAT_WRITE_BACK); + for (int i = 0; i < num_pages; i++) { + vm_page_t p = (vm_page_t)&page[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_WRITE_BACK); + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); + } - pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), - num_pages, PAT_WRITE_BACK); + return 0; +} + +static inline int set_pages_wc(struct page *page, int num_pages) +{ + //kprintf("set_pages_wc\n"); + //pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), + // num_pages, PAT_WRITE_COMBINING); + for (int i = 0; i < num_pages; i++) { + vm_page_t p = (vm_page_t)&page[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_WRITE_COMBINING); + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); + } return 0; } @@ -74,8 +110,15 @@ static inline int set_pages_wb(struct page *page, int num_pages) static inline int set_pages_array_uc(struct page **pages, int addrinarray) { - for (int i = 0; i < addrinarray; i++) - pmap_page_set_memattr((struct vm_page *)pages[i], VM_MEMATTR_UNCACHEABLE); + kprintf("set_pages_array_uc\n"); + for (int i = 0; i < addrinarray; i++) { + vm_page_t p = (vm_page_t)pages[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_UNCACHEABLE); + pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); + } return 0; } @@ -83,8 +126,15 @@ set_pages_array_uc(struct page **pages, int addrinarray) static inline int set_pages_array_wb(struct page **pages, int addrinarray) { - for (int i = 0; i < addrinarray; i++) - pmap_page_set_memattr((struct vm_page *)pages[i], VM_MEMATTR_WRITE_BACK); + //kprintf("set_pages_array_wb\n"); + for (int i = 0; i < addrinarray; i++) { + vm_page_t p = (vm_page_t)pages[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_WRITE_BACK); + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); + } return 0; } @@ -92,8 +142,15 @@ set_pages_array_wb(struct page **pages, int addrinarray) static inline int set_pages_array_wc(struct page **pages, int addrinarray) { - for (int i = 0; i < addrinarray; i++) - pmap_page_set_memattr((struct vm_page *)pages[i], VM_MEMATTR_WRITE_COMBINING); + //kprintf("set_pages_array_wc\n"); + for (int i = 0; i < addrinarray; i++) { + vm_page_t p = (vm_page_t)pages[i]; + + if (p->ext_kptr) + pmap_change_attr((vm_offset_t)p->ext_kptr, 1, + PAT_WRITE_COMBINING); + pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); + } return 0; } diff --git a/sys/dev/drm/include/asm/io.h b/sys/dev/drm/include/asm/io.h index 72e14e25b8..0a5bf3da5e 100644 --- a/sys/dev/drm/include/asm/io.h +++ b/sys/dev/drm/include/asm/io.h @@ -35,46 +35,89 @@ #include #include +/* + * Generally use atomic_read*() and atomic_write*() for these operations. + * These functions are ordered with respect to both read and write and + * both before and after the MOV. Probably vast overkill but... + */ + #undef readb static inline u8 readb(const volatile void __iomem *addr) { - return *(const volatile u8*)addr; + return atomic_read_8(__DEQUALIFY(void *, addr)); +#if 0 + u8 res; + + res = *(const volatile u8*)addr; + + return res; +#endif } #undef readw static inline u16 readw(const volatile void __iomem *addr) { - return *(const volatile u16*)addr; + return atomic_read_16(__DEQUALIFY(void *, addr)); +#if 0 + u16 res; + + res = *(const volatile u16*)addr; + + return res; +#endif } #undef readl static inline u32 readl(const volatile void __iomem *addr) { - return *(const volatile u32*)addr; + return atomic_read_32(__DEQUALIFY(void *, addr)); +#if 0 + u32 res; + + res = *(const volatile u32*)addr; + + return res; +#endif +} + +#undef readq +static inline u64 +readq(const volatile void __iomem *addr) +{ + return atomic_read_64(__DEQUALIFY(void *, addr)); } #undef writeb static inline void writeb(u8 value, volatile void __iomem *addr) { + atomic_write_8(addr, value); +#if 0 *(volatile uint8_t *)addr = value; +#endif } #undef writew static inline void writew(u16 value, volatile void __iomem *addr) { + atomic_write_16(addr, value); +#if 0 *(volatile uint16_t *)addr = value; +#endif } #undef writel static inline void writel(u32 value, volatile void __iomem *addr) { + atomic_write_32(addr, value); +#if 0 *(volatile uint32_t *)addr = value; +#endif } #define writel_relaxed(v, a) writel(v, a) @@ -83,9 +126,21 @@ writel(u32 value, volatile void __iomem *addr) static inline void writeq(u64 value, volatile void __iomem *addr) { + atomic_write_64(addr, value); +#if 0 *(volatile uint64_t *)addr = value; +#endif } +#define ioread8(addr) atomic_read_8(__DEQUALIFY(void *, addr)) +#define ioread16(addr) atomic_read_16(__DEQUALIFY(void *, addr)) +#define ioread32(addr) atomic_read_32(__DEQUALIFY(void *, addr)) + +#define iowrite8(data, addr) atomic_write_8(__DEQUALIFY(volatile void *, addr), data) +#define iowrite16(data, addr) atomic_write_16(__DEQUALIFY(volatile void *, addr), data) +#define iowrite32(data, addr) atomic_write_32(__DEQUALIFY(volatile void *, addr), data) + +#if 0 #define ioread8(addr) *(volatile uint8_t *)((char *)addr) #define ioread16(addr) *(volatile uint16_t *)((char *)addr) #define ioread32(addr) *(volatile uint32_t *)((char *)addr) @@ -104,6 +159,7 @@ writeq(u64 value, volatile void __iomem *addr) do { \ *(volatile uint32_t *)((char *)addr) = data; \ } while (0) +#endif #include @@ -146,11 +202,52 @@ ioremap_wt(resource_size_t phys_addr, unsigned long size) void iounmap(void __iomem *ptr); /* XXX these should have volatile */ + +static inline void * +memcpy_fromio(void * restrict dst, const volatile void * restrict src, + size_t len) +{ + void *res; + + cpu_mfence(); + res = memcpy(dst, src, len); + cpu_mfence(); + + return res; +} + +static inline void * +memcpy_toio(volatile void * restrict dst, const void * restrict src, + size_t len) +{ + void *res; + + cpu_mfence(); + res = memcpy(dst, src, len); + cpu_mfence(); + + return res; +} + +static inline void * +memset_io(volatile void *dst, int c, size_t len) +{ + void *res; + + cpu_mfence(); + res = memset(dst, c, len); + cpu_mfence(); + + return res; +} + +#if 0 #define memset_io(a, b, c) memset((a), (b), (c)) #define memcpy_fromio(a, b, c) memcpy((a), (b), (c)) #define memcpy_toio(a, b, c) memcpy((a), (b), (c)) +#endif -#define mmiowb cpu_sfence +#define mmiowb() __asm __volatile("sfence" : : : "memory") int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size); diff --git a/sys/dev/drm/include/asm/pgtable.h b/sys/dev/drm/include/asm/pgtable.h index 20281060c0..f6002f9b85 100644 --- a/sys/dev/drm/include/asm/pgtable.h +++ b/sys/dev/drm/include/asm/pgtable.h @@ -33,7 +33,18 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) { - return (prot | VM_MEMATTR_UNCACHEABLE); + return (prot | _PAGE_PCD | _PAGE_PWT); +} + +static inline int pgflags_to_memattr(pgprot_t prot) +{ + switch (prot & _PAGE_CACHE_MASK) { + case _PAGE_PAT | _PAGE_PCD: + return VM_MEMATTR_WRITE_COMBINING; + case _PAGE_PCD | _PAGE_PWT: + return VM_MEMATTR_UNCACHEABLE; + } + return VM_MEMATTR_WRITE_BACK; } #endif /* _ASM_PGTABLE_H_ */ diff --git a/sys/dev/drm/include/asm/pgtable_types.h b/sys/dev/drm/include/asm/pgtable_types.h index 202922187f..291927aa3f 100644 --- a/sys/dev/drm/include/asm/pgtable_types.h +++ b/sys/dev/drm/include/asm/pgtable_types.h @@ -54,7 +54,7 @@ static inline pgprot_t pgprot_writecombine(pgprot_t prot) { - return (prot | VM_MEMATTR_WRITE_COMBINING); + return (prot | _PAGE_PAT | _PAGE_PCD); } #define __pgprot(value) ((pgprot_t) {(value)}) diff --git a/sys/dev/drm/include/drm/ttm/ttm_bo_driver.h b/sys/dev/drm/include/drm/ttm/ttm_bo_driver.h index 5003e4cfc7..bcd2008357 100644 --- a/sys/dev/drm/include/drm/ttm/ttm_bo_driver.h +++ b/sys/dev/drm/include/drm/ttm/ttm_bo_driver.h @@ -788,7 +788,11 @@ static inline void ttm_bo_unreserve(struct ttm_buffer_object *bo) { if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { lockmgr(&bo->bdev->glob->lru_lock, LK_EXCLUSIVE); - ttm_bo_add_to_lru(bo); + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { + ttm_bo_add_to_lru(bo); + } else { + kprintf("ttm_bo_unreserve: FLAG_NO_EVICT race\n"); + } lockmgr(&bo->bdev->glob->lru_lock, LK_RELEASE); } __ttm_bo_unreserve(bo); diff --git a/sys/dev/drm/include/drm/ttm/ttm_page_alloc.h b/sys/dev/drm/include/drm/ttm/ttm_page_alloc.h index 4d9b019d25..5d084d17f6 100644 --- a/sys/dev/drm/include/drm/ttm/ttm_page_alloc.h +++ b/sys/dev/drm/include/drm/ttm/ttm_page_alloc.h @@ -31,6 +31,12 @@ struct device; +/** + * Set caching mode for an array of pages + */ +int ttm_set_pages_caching(struct page **pages, + enum ttm_caching_state cstate, unsigned cpages); + /** * Initialize pool allocator. */ diff --git a/sys/dev/drm/include/drm/ttm/ttm_set_memory.h b/sys/dev/drm/include/drm/ttm/ttm_set_memory.h index 7c492b49e3..9593f7d985 100644 --- a/sys/dev/drm/include/drm/ttm/ttm_set_memory.h +++ b/sys/dev/drm/include/drm/ttm/ttm_set_memory.h @@ -57,17 +57,28 @@ static inline int ttm_set_pages_wb(struct page *page, int numpages) return set_pages_wb(page, numpages); } +/* + * Removed + * + * https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=db9c1734ad69c0ba5e5e420ba31ebc1048976be6 + */ +#if 1 /* ZZZ */ static inline int ttm_set_pages_wc(struct page *page, int numpages) { + return set_pages_wc(page, numpages); +#if 0 + /* this is broken, returns physical address, not VA */ unsigned long addr = (unsigned long)page_address(page); return set_memory_wc(addr, numpages); +#endif } static inline int ttm_set_pages_uc(struct page *page, int numpages) { return set_pages_uc(page, numpages); } +#endif #else /* for CONFIG_X86 */ @@ -135,6 +146,12 @@ static inline int ttm_set_pages_wb(struct page *page, int numpages) #endif /* for CONFIG_AGP */ +/* + * Removed + * + * https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=db9c1734ad69c0ba5e5e420ba31ebc1048976be6 + */ +#if 1 /* ZZZ */ static inline int ttm_set_pages_wc(struct page *page, int numpages) { return 0; @@ -144,6 +161,7 @@ static inline int ttm_set_pages_uc(struct page *page, int numpages) { return 0; } +#endif #endif /* for CONFIG_X86 */ diff --git a/sys/dev/drm/include/linux/dma-fence.h b/sys/dev/drm/include/linux/dma-fence.h index 81c78144d2..65a4faba5a 100644 --- a/sys/dev/drm/include/linux/dma-fence.h +++ b/sys/dev/drm/include/linux/dma-fence.h @@ -182,8 +182,8 @@ static inline void dma_fence_set_error(struct dma_fence *fence, int error) { fence->error = error; -kprintf("fence error: context=%llx, seqno=%d, error=%d\n", fence->context, fence->seqno, error); -print_backtrace(-1); +kprintf("fence error: fence=%p context=0x%llx, seqno=%d, error=%d\n", fence, fence->context, fence->seqno, error); +//print_backtrace(-1); } static inline struct dma_fence * diff --git a/sys/dev/drm/include/linux/gfp.h b/sys/dev/drm/include/linux/gfp.h index f7efb8e2f7..3ee633a8a2 100644 --- a/sys/dev/drm/include/linux/gfp.h +++ b/sys/dev/drm/include/linux/gfp.h @@ -29,10 +29,12 @@ #include #include +#include #include #include #include +#include #include #define GFP_NOWAIT (M_NOWAIT | M_CACHEALIGN) @@ -58,17 +60,34 @@ static inline void __free_page(struct page *page) { - vm_page_freezwq((vm_page_t)page); + vm_page_t m = (vm_page_t)page; + void *kptr; + + KKASSERT(m->ext_refs == 0); + if ((kptr = m->ext_kptr) != NULL) { + m->ext_kptr = NULL; + pmap_qremove((vm_offset_t)kptr, 1); + kmem_free(kernel_map, (vm_offset_t)kptr, PAGE_SIZE); + } + vm_page_free_contig((struct vm_page *)page, PAGE_SIZE); } -static inline struct page * alloc_page(int flags) +static inline struct page * +alloc_page(int flags) { vm_paddr_t high = BUS_SPACE_MAXADDR; + size_t bytes = PAGE_SIZE; + vm_page_t p; if (flags & GFP_DMA32) high = BUS_SPACE_MAXADDR_32BIT; - return (struct page *)vm_page_alloczwq(0, VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM | - VM_ALLOC_INTERRUPT); + + p = vm_page_alloc_contig(0LLU, high, bytes, bytes, bytes, + VM_MEMATTR_DEFAULT); + if (p && (flags & __GFP_ZERO)) { + bzero((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(p)), PAGE_SIZE); + } + return ((struct page *)p); } static inline bool @@ -83,15 +102,23 @@ gfpflags_allow_blocking(const gfp_t flags) * to restrict the address space, so neither do we. */ static inline struct page * -alloc_pages(gfp_t gfp_mask, unsigned int order) +alloc_pages(gfp_t flags, unsigned int order) { - size_t bytes = PAGE_SIZE << order; + size_t bytes = (size_t)PAGE_SIZE << order; struct vm_page *pgs; + vm_paddr_t high = BUS_SPACE_MAXADDR; - pgs = vm_page_alloc_contig(0LLU, ~0LLU, bytes, bytes, bytes, + if (flags & GFP_DMA32) + high = BUS_SPACE_MAXADDR_32BIT; + + /* lo, hi, align, boundary, size, memattr */ + pgs = vm_page_alloc_contig(0LLU, high, bytes, bytes, bytes, VM_MEMATTR_DEFAULT); + if (pgs && (flags & __GFP_ZERO)) { + bzero((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pgs)), bytes); + } - return (struct page*)pgs; + return ((struct page*)pgs); } /* @@ -100,9 +127,22 @@ alloc_pages(gfp_t gfp_mask, unsigned int order) static inline void __free_pages(struct page *pgs, unsigned int order) { - size_t bytes = PAGE_SIZE << order; - - vm_page_free_contig((struct vm_page *)pgs, bytes); + size_t pgcount = 1 << order; + size_t i; + vm_page_t m; + void *kptr; + + for (i = 0; i < pgcount; ++i) { + m = (vm_page_t)&pgs[i]; + + KKASSERT(m->ext_refs == 0); + if ((kptr = m->ext_kptr) != NULL) { + m->ext_kptr = NULL; + pmap_qremove((vm_offset_t)kptr, 1); + kmem_free(kernel_map, (vm_offset_t)kptr, PAGE_SIZE); + } + } + vm_page_free_contig((struct vm_page *)pgs, PAGE_SIZE * pgcount); } #endif /* _LINUX_GFP_H_ */ diff --git a/sys/dev/drm/include/linux/highmem.h b/sys/dev/drm/include/linux/highmem.h index c016d5fc48..ac17a930ed 100644 --- a/sys/dev/drm/include/linux/highmem.h +++ b/sys/dev/drm/include/linux/highmem.h @@ -37,6 +37,8 @@ #include +#include + static inline struct page * kmap_to_page(void *addr) { @@ -48,28 +50,123 @@ kmap_to_page(void *addr) static inline void *kmap(struct page *pg) { - return (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( (struct vm_page *)pg )); + vm_page_t ary[1]; + void *kptr; + + ary[0] = &pg->pa_vmpage; + atomic_add_int(&pg->pa_vmpage.ext_refs, 1); + while ((kptr = pg->pa_vmpage.ext_kptr) == NULL) { + kptr = (void *)kmem_alloc_nofault(kernel_map, + PAGE_SIZE, + VM_SUBSYS_DRM_VMAP, + PAGE_SIZE); + //pmap_qenter((vm_offset_t)kptr, ary, 1); + pmap_qenter_memattr((vm_offset_t)kptr, ary, 1, + VM_MEMATTR_UNCACHEABLE); + if (atomic_cmpset_ptr(&pg->pa_vmpage.ext_kptr, NULL, kptr)) + break; + pmap_qremove(kptr, 1); + kmem_free(kernel_map, kptr, PAGE_SIZE); + } + return kptr; } static inline void kunmap(struct page *pg) { - /* Nothing to do on systems with a direct memory map */ +#if 0 + atomic_add_int(&pg->pa_vmpage.ext_refs, -1); + /* leave kptr cached */ +#else + void *kptr; + if (atomic_fetchadd_int(&pg->pa_vmpage.ext_refs, -1) == 1) { + kptr = atomic_swap_ptr((void *)&pg->pa_vmpage.ext_kptr, NULL); + if (kptr) { + pmap_qremove(kptr, 1); + kmem_free(kernel_map, kptr, PAGE_SIZE); + } + } +#endif } +/* + * kmap_atomic() / kunmap_atomic() maps a page to kernel memory. Because + * kunmap_atomic() stupidly takes only a memory pointer, we must allocate + * custom space for each call. + * + * It does not appear that callers expect the returned area to always + * be in a special cache mode. + * + * It doesn't look like we can assume cpu-localized map/unmap here, + * use a regular qenter. + */ static inline void *kmap_atomic(struct page *pg) { - return (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS( (struct vm_page *)pg )); + vm_page_t ary[1]; + void *kptr; + + ary[0] = &pg->pa_vmpage; + kptr = (void *)kmem_alloc_nofault(kernel_map, + PAGE_SIZE, + VM_SUBSYS_DRM_VMAP, + PAGE_SIZE); + pmap_qenter/*_quick[_memattr]*/((vm_offset_t)kptr, ary, 1); + //pmap_qenter_memattr(kptr, ary, 1, VM_MEMATTR_UNCACHEABLE); /* YYY */ + return kptr; } +/* + * NOTE: prot is pgflags + * NOTE: use case is cpu-localized + */ static inline void * kmap_atomic_prot(struct page *pg, pgprot_t prot) { - return kmap_atomic(pg); + vm_page_t ary[1]; + void *kptr; + + ary[0] = &pg->pa_vmpage; + kptr = (void *)kmem_alloc_nofault(kernel_map, + PAGE_SIZE, + VM_SUBSYS_DRM_VMAP, + PAGE_SIZE); + pmap_qenter_quick_memattr((vm_offset_t)kptr, ary, 1, + pgflags_to_memattr(prot)); + //VM_MEMATTR_UNCACHEABLE); /* YYY */ + return kptr; +} + +static inline void +kunmap_atomic(void *vaddr) +{ + pmap_qremove(vaddr, 1); + kmem_free(kernel_map, vaddr, PAGE_SIZE); +} + +/* + * Quickly map a page for temporary use on the current cpu for a short while, + * using the given memory attribute. + * + * NOTE: This enters a critical section + */ +static inline void * +kmap_atomic_quick(struct page *pg, pgprot_t prot) +{ + vm_page_t m = (vm_page_t)pg; + + return ((void *)pmap_tempmap_enter(VM_PAGE_TO_PHYS(m), + pgflags_to_memattr(prot))); + //VM_MEMATTR_UNCACHEABLE)); /* YYY */ } -static inline void kunmap_atomic(void *vaddr) +/* + * Undo the effects of kmap_atomic_quick(). + * + * NOTE: This exits a critical section. + */ +static inline void +kunmap_atomic_quick(void) { - /* Nothing to do on systems with a direct memory map */ + pmap_tempmap_exit(); } #endif /* _LINUX_HIGHMEM_H_ */ diff --git a/sys/dev/drm/include/linux/io-mapping.h b/sys/dev/drm/include/linux/io-mapping.h index 9d069f5dc7..ec91c4794b 100644 --- a/sys/dev/drm/include/linux/io-mapping.h +++ b/sys/dev/drm/include/linux/io-mapping.h @@ -64,6 +64,7 @@ static inline void io_mapping_free(struct io_mapping *mapping) { /* Default memory attribute is write-back */ pmap_mapdev_attr(mapping->base, mapping->size, VM_MEMATTR_WRITE_BACK); + //pmap_unmapdev(mapping->vaddr, mapping->size); kfree(mapping); } diff --git a/sys/dev/drm/include/linux/lockdep.h b/sys/dev/drm/include/linux/lockdep.h index 3a4eabc5bf..9f9d7f4729 100644 --- a/sys/dev/drm/include/linux/lockdep.h +++ b/sys/dev/drm/include/linux/lockdep.h @@ -38,6 +38,12 @@ lockdep_assert_held(struct lock *l) KKASSERT(lockinuse(l)); } +static inline int +lockdep_is_held(struct lock *l) +{ + return(lockstatus(l, curthread) == LK_EXCLUSIVE); +} + #define might_lock(lock) do { } while (0) struct lock_class_key { diff --git a/sys/dev/drm/include/linux/preempt.h b/sys/dev/drm/include/linux/preempt.h index f276bc34f9..ac335559e5 100644 --- a/sys/dev/drm/include/linux/preempt.h +++ b/sys/dev/drm/include/linux/preempt.h @@ -29,10 +29,21 @@ #include #include +#include +#include +#include +#if 1 +#define preempt_disable() crit_enter(); +#define preempt_enable() crit_exit(); +#else #define preempt_disable() cpu_ccfence() - #define preempt_enable() cpu_ccfence() +#endif + +#define in_interrupt() ((curthread->td_flags & TDF_INTTHREAD) || \ + IN_CRITICAL_SECT(curthread) || \ + mycpu->gd_intr_nesting_level) #define in_atomic() (curthread->td_flags & TDF_NOFAULT) diff --git a/sys/dev/drm/include/linux/rcupdate.h b/sys/dev/drm/include/linux/rcupdate.h index b04dd4cb1d..0adaf88d80 100644 --- a/sys/dev/drm/include/linux/rcupdate.h +++ b/sys/dev/drm/include/linux/rcupdate.h @@ -23,6 +23,9 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +/* + * See: https://www.kernel.org/doc/Documentation/RCU/whatisRCU.txt + */ #ifndef _LINUX_RCUPDATE_H_ #define _LINUX_RCUPDATE_H_ @@ -43,34 +46,63 @@ #include +#include +#include + +#define RCU_WARN_ONCE(c, ...) do { \ + WARN_ONCE((c), ##__VA_ARGS__); \ +} while(0) + +#define __rcu_var_name(n, f, l) \ + __CONCAT(__CONCAT(__CONCAT(rcu_, n), _), __COUNTER__) + +#if 0 static inline void rcu_read_lock(void) { - preempt_disable(); + exis_hold(); } static inline void rcu_read_unlock(void) { - preempt_enable(); + exis_drop(); } +#endif -#define rcu_dereference_protected(p, condition) \ - ((typeof(*p) *)(p)) - -#define rcu_dereference(p) \ +#define __rcu_dereference_protected(p, c, n) \ ({ \ - typeof(*(p)) *__rcu_dereference_tmp = READ_ONCE(p); \ - __rcu_dereference_tmp; \ + RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ + __func__, __LINE__, __XSTRING(n)); \ + rcu_dereference(p); \ }) +#define rcu_dereference_protected(p, c) \ + __rcu_dereference_protected((p), (c), \ + __rcu_var_name(protected, __func__, __LINE__)) + +#define __rcu_dereference_check(p, c, n) \ +({ \ + __typeof(*p) *n = rcu_dereference(p); \ + RCU_WARN_ONCE(!(c), "%s:%d: condition for %s failed\n", \ + __func__, __LINE__, __XSTRING(n)); \ + n; \ +}) + +#define rcu_dereference_check(p, c) \ + __rcu_dereference_check((p), (c) || rcu_read_lock_held(), \ + __rcu_var_name(check, __func__, __LINE__)) + +#define rcu_dereference(p) \ + ((__typeof(*p) *)READ_ONCE(p)) + #define rcu_dereference_raw(p) \ ((__typeof(*p) *)READ_ONCE(p)) -#define rcu_assign_pointer(p, v) \ -do { \ - cpu_mfence(); \ - WRITE_ONCE((p), (v)); \ +#define rcu_assign_pointer(p, v) \ +do { \ + atomic_store_rel_ptr((volatile uintptr_t *)&(p), \ + (uintptr_t)(v)); \ } while (0) #define RCU_INIT_POINTER(p, v) \ @@ -79,6 +111,8 @@ do { \ } while (0) extern void __kfree_rcu(void *ptr); +extern void rcu_read_lock(void); +extern void rcu_read_unlock(void); #define kfree_rcu(ptr, rcu_head) \ do { \ diff --git a/sys/dev/drm/include/linux/rcutree.h b/sys/dev/drm/include/linux/rcutree.h index f90cdc17a9..96e42d5911 100644 --- a/sys/dev/drm/include/linux/rcutree.h +++ b/sys/dev/drm/include/linux/rcutree.h @@ -27,12 +27,7 @@ #ifndef _LINUX_RCUTREE_H_ #define _LINUX_RCUTREE_H_ -static inline void rcu_barrier(void) {} - -static inline void -synchronize_rcu_expedited(void) -{ - cpu_mfence(); -} +void rcu_barrier(void); +void synchronize_rcu_expedited(void); #endif /* _LINUX_RCUTREE_H_ */ diff --git a/sys/dev/drm/include/linux/seqlock.h b/sys/dev/drm/include/linux/seqlock.h index dae4d61ae0..578e940454 100644 --- a/sys/dev/drm/include/linux/seqlock.h +++ b/sys/dev/drm/include/linux/seqlock.h @@ -70,14 +70,14 @@ static inline void write_seqlock(seqlock_t *sl) { spin_lock(&sl->lock); - sl->sequence++; + atomic_add_int(&sl->sequence, 1); cpu_sfence(); } static inline void write_sequnlock(seqlock_t *sl) { - sl->sequence--; + atomic_add_int(&sl->sequence, 1); spin_unlock(&sl->lock); cpu_sfence(); } @@ -121,6 +121,7 @@ __read_seqcount_begin(const seqcount_t *s) if ((ret & 1) == 0) break; cpu_pause(); + cpu_lfence(); } while (1); return ret; @@ -151,14 +152,14 @@ read_seqcount_retry(const seqcount_t *s, unsigned start) static inline void write_seqcount_begin(seqcount_t *s) { - s->sequence++; + atomic_add_int(&s->sequence, 1); cpu_ccfence(); } static inline void write_seqcount_end(seqcount_t *s) { cpu_ccfence(); - s->sequence++; + atomic_add_int(&s->sequence, 1); } static inline unsigned int @@ -174,17 +175,17 @@ raw_read_seqcount(const seqcount_t *s) static inline void write_seqlock_irqsave(seqlock_t *sl, unsigned long flags) { - local_irq_save(flags); spin_lock(&sl->lock); - sl->sequence++; + local_irq_save(flags); + atomic_add_int(&sl->sequence, 1); } static inline void write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) { - sl->sequence--; - spin_unlock(&sl->lock); + atomic_add_int(&sl->sequence, 1); local_irq_restore(flags); + spin_unlock(&sl->lock); } #endif /* _LINUX_SEQLOCK_H_ */ diff --git a/sys/dev/drm/include/linux/spinlock.h b/sys/dev/drm/include/linux/spinlock.h index f36504ce59..50fa00f6fd 100644 --- a/sys/dev/drm/include/linux/spinlock.h +++ b/sys/dev/drm/include/linux/spinlock.h @@ -56,14 +56,14 @@ static inline void spin_lock_irq(spinlock_t *lock) { local_irq_disable(); - lockmgr(lock, LK_EXCLUSIVE); + lockmgr(lock, LK_EXCLUSIVE | LK_SPIN); } static inline int spin_trylock_irq(spinlock_t *lock) { local_irq_disable(); preempt_disable(); - return lockmgr_try(lock, LK_EXCLUSIVE); + return lockmgr_try(lock, LK_EXCLUSIVE | LK_SPIN); } static inline void spin_unlock_irq(spinlock_t *lock) @@ -75,7 +75,7 @@ static inline void spin_unlock_irq(spinlock_t *lock) #define spin_lock_irqsave(lock, flags) \ ({ \ local_irq_save(flags); \ - lockmgr(lock, LK_EXCLUSIVE); \ + lockmgr(lock, LK_EXCLUSIVE | LK_SPIN); \ }) static inline void @@ -86,21 +86,22 @@ spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) } /* - XXX: the spin_lock_bh() and spin_unlock_bh() functions are possibly incorrect - XXX: see also in_interrupt() -*/ + * In linux, the spin*_bh() functions disable softirqs but not hard + * interrupts. In DragonFly, softirqs cannot interrupt kernel code + * anyway. + */ static inline void spin_lock_bh(struct lock *lock) { - crit_enter(); - lockmgr(lock, LK_EXCLUSIVE); + //crit_enter(); + lockmgr(lock, LK_EXCLUSIVE | LK_SPIN); } static inline void spin_unlock_bh(struct lock *lock) { lockmgr(lock, LK_RELEASE); - crit_exit(); + //crit_exit(); } #define DEFINE_SPINLOCK(x) struct lock x = LOCK_INITIALIZER("ds##x", 0, 0) diff --git a/sys/dev/drm/include/linux/wait.h b/sys/dev/drm/include/linux/wait.h index b13a3bb7f3..6e3de0c7b7 100644 --- a/sys/dev/drm/include/linux/wait.h +++ b/sys/dev/drm/include/linux/wait.h @@ -127,25 +127,18 @@ void finish_wait(wait_queue_head_t *q, wait_queue_entry_t *wait); if (condition) \ break; \ \ - tsleep_interlock(current, flags); \ + tsleep_interlock(&wq, flags); \ \ - if ((timeout_jiffies) != 0) { \ - ret = tsleep(current, PINTERLOCKED|flags, "lwe", timeout_jiffies); \ - } else { \ - ret = tsleep(current, PINTERLOCKED|flags, "lwe", hz);\ - if (ret == EWOULDBLOCK) { \ - /*kprintf("F");*/ \ - /*print_backtrace(-1);*/ \ - ret = 0; \ - } \ - } \ + if (condition) \ + break; \ + ret = tsleep(&wq, PINTERLOCKED|flags, "lwe", timeout_jiffies);\ \ if (ret == EINTR || ret == ERESTART) { \ interrupted = true; \ break; \ } \ if (ret == EWOULDBLOCK) { \ - timeout_expired = true; \ + timeout_expired = true; \ break; \ } \ } \ @@ -165,7 +158,7 @@ void finish_wait(wait_queue_head_t *q, wait_queue_entry_t *wait); retval = 1; \ \ finish_wait(&wq, &tmp_wq); \ - retval; \ + retval; \ }) #define wait_event(wq, condition) \ diff --git a/sys/dev/drm/include/linux/ww_mutex.h b/sys/dev/drm/include/linux/ww_mutex.h index 1cb7122c81..75d5cdacb1 100644 --- a/sys/dev/drm/include/linux/ww_mutex.h +++ b/sys/dev/drm/include/linux/ww_mutex.h @@ -82,6 +82,7 @@ extern int ww_mutex_lock_interruptible(struct ww_mutex *ww, struct ww_acquire_ctx *ctx); extern int ww_mutex_lock_slow_interruptible(struct ww_mutex *ww, struct ww_acquire_ctx *ctx); +extern int ww_mutex_lock_recursive(struct ww_mutex *ww); extern void ww_mutex_unlock(struct ww_mutex *ww); extern void ww_mutex_destroy(struct ww_mutex *ww); @@ -102,6 +103,8 @@ ww_mutex_is_locked(struct ww_mutex *ww) static inline int ww_mutex_trylock(struct ww_mutex *ww) { + if (lockstatus(&ww->base, NULL) == LK_EXCLUSIVE) + return 0; return (lockmgr(&ww->base, LK_EXCLUSIVE|LK_NOWAIT) == 0); } diff --git a/sys/dev/drm/linux_dma-buf.c b/sys/dev/drm/linux_dma-buf.c index 07482469cb..b9f8b1a746 100644 --- a/sys/dev/drm/linux_dma-buf.c +++ b/sys/dev/drm/linux_dma-buf.c @@ -60,6 +60,9 @@ dmabuf_close(struct file *fp) return EINVAL; } dmabuf = fp->private_data; + fp->f_ops = &badfileops; + fp->f_data = NULL; + fp->private_data = NULL; dmabuf->ops->release(dmabuf); kfree(dmabuf); @@ -147,20 +150,19 @@ dma_buf_fd(struct dma_buf *dmabuf, int flags) if (dmabuf->file == NULL) return -EINVAL; - if (flags & O_CLOEXEC) { - /* XXX: CLOEXEC not handled yet */ -#if 0 - __set_close_on_exec(fd, fdt); - else - __clear_close_on_exec(fd, fdt); -#endif - } - error = fdalloc(curproc, 0, &fd); - if (error != 0) + if (error) return -error; + if (flags & O_CLOEXEC) + curproc->p_fd->fd_files[fd].fileflags |= UF_EXCLOSE; + + /* + * fsetfd() installs the fp and adds a ref. We must + * drop the ref we obtained from fdalloc() to finish up. + */ fsetfd(curproc->p_fd, dmabuf->file, fd); + fdrop(dmabuf->file); return fd; } @@ -181,6 +183,7 @@ dma_buf_get(int fd) } dmabuf = fp->private_data; + fhold(fp); /* for dma_buf_put() */ dropfp(curthread, fd, fp); return dmabuf; diff --git a/sys/dev/drm/linux_fence.c b/sys/dev/drm/linux_fence.c index 3201632698..c7defd8c33 100644 --- a/sys/dev/drm/linux_fence.c +++ b/sys/dev/drm/linux_fence.c @@ -72,7 +72,7 @@ dma_fence_context_alloc(unsigned num) struct default_wait_cb { struct dma_fence_cb base; - struct task_struct *task; + void *wake_id; }; static void @@ -81,14 +81,15 @@ dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb) struct default_wait_cb *wait = container_of(cb, struct default_wait_cb, base); - wake_up_process(wait->task); + //wake_up_process(wait->task); + wakeup(wait->wake_id); } long dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) { long ret = timeout ? timeout : 1; - unsigned long end; + int wake_id = 0; int err; struct default_wait_cb cb; bool was_set; @@ -118,33 +119,27 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) } cb.base.func = dma_fence_default_wait_cb; - cb.task = current; + cb.wake_id = &wake_id; list_add(&cb.base.node, &fence->cb_list); - end = jiffies + timeout; - for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) { - if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { - break; - } - if (intr) { - __set_current_state(TASK_INTERRUPTIBLE); - } - else { - __set_current_state(TASK_UNINTERRUPTIBLE); - } - crit_exit(); - /* wake_up_process() directly uses task_struct pointers as sleep identifiers */ - err = lksleep(current, fence->lock, intr ? PCATCH : 0, "dmafence", ret); - crit_enter(); + tsleep_interlock(&wake_id, (intr ? PCATCH : 0)); + while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + /* can sleep with a crit section held */ + err = lksleep(&wake_id, fence->lock, + (intr ? PCATCH : 0) | PINTERLOCKED, + "dmafence", timeout); if (err == EINTR || err == ERESTART) { ret = -ERESTARTSYS; break; + } else if (err == EWOULDBLOCK) { + ret = 0; + break; } + tsleep_interlock(&wake_id, (intr ? PCATCH : 0)); } if (!list_empty(&cb.base.node)) list_del(&cb.base.node); - __set_current_state(TASK_RUNNING); out: crit_exit(); lockmgr(fence->lock, LK_RELEASE); @@ -153,7 +148,7 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) static bool dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count, - uint32_t *idx) + uint32_t *idx) { int i; @@ -170,10 +165,11 @@ dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count, long dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, - bool intr, long timeout, uint32_t *idx) + bool intr, long timeout, uint32_t *idx) { struct default_wait_cb *cb; long ret = timeout; + int wake_id = 0; unsigned long end; int i, err; @@ -194,9 +190,10 @@ dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, for (i = 0; i < count; i++) { struct dma_fence *fence = fences[i]; - cb[i].task = current; + cb[i].wake_id = &wake_id; if (dma_fence_add_callback(fence, &cb[i].base, - dma_fence_default_wait_cb)) { + dma_fence_default_wait_cb)) + { if (idx) *idx = i; goto cb_cleanup; @@ -205,9 +202,11 @@ dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, end = jiffies + timeout; for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) { + tsleep_interlock(&wake_id, (intr ? PCATCH : 0)); if (dma_fence_test_signaled_any(fences, count, idx)) break; - err = tsleep(current, intr ? PCATCH : 0, "dfwat", ret); + err = tsleep(&wake_id, (intr ? PCATCH : 0) | PINTERLOCKED, + "dfwat", ret); if (err == EINTR || err == ERESTART) { ret = -ERESTARTSYS; break; @@ -218,12 +217,34 @@ dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, while (i-- > 0) dma_fence_remove_callback(fences[i], &cb[i].base); kfree(cb); + return ret; } int dma_fence_signal_locked(struct dma_fence *fence) { +#if 1 + struct dma_fence_cb *cur, *tmp; + int ret = 0; + + if (fence == NULL) + return -EINVAL; + + if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + ret = -EINVAL; + } else { + fence->timestamp = ktime_get(); + set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); + } + + list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { + INIT_LIST_HEAD(&cur->node); + cur->func(fence, cur); + } + + return ret; +#else struct dma_fence_cb *cur, *tmp; struct list_head cb_list; @@ -244,11 +265,35 @@ dma_fence_signal_locked(struct dma_fence *fence) } return 0; +#endif } int dma_fence_signal(struct dma_fence *fence) { +#if 1 + struct dma_fence_cb *cur, *tmp; + + if (fence == NULL) + return -EINVAL; + + if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + return -EINVAL; + } + + fence->timestamp = ktime_get(); + set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); + + if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) { + lockmgr(fence->lock, LK_EXCLUSIVE); + list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) { + INIT_LIST_HEAD(&cur->node); + cur->func(fence, cur); + } + lockmgr(fence->lock, LK_RELEASE); + } + return 0; +#else int r; if (fence == NULL) @@ -261,6 +306,7 @@ dma_fence_signal(struct dma_fence *fence) crit_exit(); return r; +#endif } void @@ -269,12 +315,12 @@ dma_fence_enable_sw_signaling(struct dma_fence *fence) if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && fence->ops->enable_signaling) { - crit_enter(); + //crit_enter(); lockmgr(fence->lock, LK_EXCLUSIVE); if (!fence->ops->enable_signaling(fence)) dma_fence_signal_locked(fence); lockmgr(fence->lock, LK_RELEASE); - crit_exit(); + //crit_exit(); } } diff --git a/sys/dev/drm/linux_rcu.c b/sys/dev/drm/linux_rcu.c index 72509de0d4..b56dfa19be 100644 --- a/sys/dev/drm/linux_rcu.c +++ b/sys/dev/drm/linux_rcu.c @@ -39,7 +39,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -47,8 +49,7 @@ #include typedef struct rcu_elm { - enum { RCU_NULL, RCU_CALL, RCU_FREE } type; - int ticks; + enum { RCU_NULL, RCU_CALL } type; void (*func)(struct rcu_head *arg); void *ptr; } rcu_elm_t; @@ -60,10 +61,62 @@ typedef struct rcu_pcpu { int s; int e; int running; + int unused01; struct callout timer_callout; } rcu_pcpu_t; -static rcu_pcpu_t *rcupcpu; +static rcu_pcpu_t rcus; + +/* + * If this winds up being too much of a mess, just punt and use this + * to actually implement RCU + */ +struct lock linux_rcu_lock = LOCK_INITIALIZER("rculk", 0, LK_CANRECURSE); + +void +rcu_read_lock(void) +{ + lockmgr(&linux_rcu_lock, LK_SHARED); + //exis_hold(); +} + +void +rcu_read_unlock(void) +{ + //exis_drop(); + lockmgr(&linux_rcu_lock, LK_RELEASE); +} + +static void +rcu_callback_barrier(struct rcu_head *ptr) +{ + *(volatile long *)ptr = 1; + wakeup(ptr); +} + +void +rcu_barrier(void) +{ + volatile long wval = 0; + + call_rcu((void *)&wval, rcu_callback_barrier); + while (wval == 0) + tsleep(&wval, 0, "rcubar", hz); +} + +#if 0 +static inline void +synchronize_rcu_expedited(void) +{ + rcu_barrier(); +} +#endif + +static void +rcu_callback_kfree(struct rcu_head *ptr) +{ + kfree(ptr); +} /* * Timer callout (pcpu) @@ -73,13 +126,134 @@ rcu_timer(void *arg) { rcu_pcpu_t *rcu = arg; rcu_elm_t *elm; - int delta; + + if (lockmgr(&linux_rcu_lock, LK_EXCLUSIVE | LK_NOWAIT)) { + callout_reset_bycpu(&rcu->timer_callout, 1, rcu_timer, + rcu, mycpuid); + return; + } + + while (rcu->s != rcu->e) { + elm = &rcu->elms[rcu->s & rcu->mask]; + + lockmgr(&linux_rcu_lock, LK_RELEASE); + + switch(elm->type) { + case RCU_NULL: + break; + case RCU_CALL: + elm->func(elm->ptr); + break; + } + lockmgr(&linux_rcu_lock, LK_EXCLUSIVE); + + elm->type = RCU_NULL; + ++rcu->s; + } + if (rcu->s == rcu->e) { + rcu->running = 0; + } else { + callout_reset_bycpu(&rcu->timer_callout, 1, rcu_timer, + rcu, mycpuid); + } + lockmgr(&linux_rcu_lock, LK_RELEASE); +} + +/* + * Expand the rcu array for the current cpu + */ +static void +rcu_expand(rcu_pcpu_t *rcu) +{ + rcu_elm_t *oelms; + rcu_elm_t *nelms; + int count; + int nsize; + int nmask; + int n; + + count = rcu->e - rcu->s; /* note: 2s complement underflow */ + while (unlikely(count == rcu->size)) { + nsize = count ? count * 2 : 16; + nelms = kzalloc(nsize * sizeof(*nelms), GFP_KERNEL); + kprintf("drm: expand RCU cpu %d to %d\n", mycpuid, nsize); + if (likely(count == rcu->size)) { + nmask = nsize - 1; + oelms = rcu->elms; + n = rcu->s; + while (n != rcu->e) { + nelms[n & nmask] = oelms[n & rcu->mask]; + ++n; + } + rcu->elms = nelms; + rcu->size = nsize; + rcu->mask = nmask; + nelms = oelms; + } + if (likely(nelms != NULL)) + kfree(nelms); + count = rcu->e - rcu->s; + } + KKASSERT(count >= 0 && count < rcu->size); +} + +void +call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *)) +{ + rcu_pcpu_t *rcu; + rcu_elm_t *elm; + + rcu = &rcus; + + lockmgr(&linux_rcu_lock, LK_EXCLUSIVE); + + rcu_expand(rcu); + elm = &rcu->elms[rcu->e & rcu->mask]; + ++rcu->e; + + elm->type = RCU_CALL; + elm->func = func; + elm->ptr = head; + + if (rcu->running == 0) { + rcu->running = 1; + callout_reset_bycpu(&rcu->timer_callout, 1, rcu_timer, + rcu, mycpuid); + } + lockmgr(&linux_rcu_lock, LK_RELEASE); +} + +void +__kfree_rcu(void *ptr) +{ + call_rcu(ptr, rcu_callback_kfree); +} + +static int +init_rcu(void *dummy __unused) +{ + callout_init_mp(&rcus.timer_callout); + + return 0; +} + +#if 0 +/* + * Timer callout (pcpu) + */ +static void +rcu_timer(void *arg) +{ + rcu_pcpu_t *rcu = arg; + rcu_elm_t *elm; + long delta; crit_enter(); while (rcu->s != rcu->e) { elm = &rcu->elms[rcu->s & rcu->mask]; - delta = ticks - elm->ticks; /* 2s compl underflow */ - if (delta < hz) + + delta = pseudo_ticks - elm->pticks; + if (delta < 1) break; switch(elm->type) { @@ -98,7 +272,7 @@ rcu_timer(void *arg) if (rcu->s == rcu->e) { rcu->running = 0; } else { - callout_reset_bycpu(&rcu->timer_callout, hz / 10, rcu_timer, + callout_reset_bycpu(&rcu->timer_callout, 1, rcu_timer, rcu, mycpuid); } crit_exit(); @@ -159,7 +333,7 @@ rcu_expand(rcu_pcpu_t *rcu) } void -__kfree_rcu(void *ptr) +kfree_rcu(void *ptr, void *rcu __unused) { rcu_pcpu_t *rcu; rcu_elm_t *elm; @@ -176,7 +350,7 @@ __kfree_rcu(void *ptr) ++rcu->e; elm->type = RCU_FREE; - elm->ticks = ticks; + elm->pticks = pseudo_ticks; elm->ptr = ptr; rcu_ping(rcu); @@ -201,7 +375,7 @@ call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *)) ++rcu->e; elm->type = RCU_CALL; - elm->ticks = ticks; + elm->pticks = pseudo_ticks; elm->func = func; elm->ptr = head; @@ -220,5 +394,6 @@ init_rcu(void *dummy __unused) } return 0; } +#endif SYSINIT(linux_rcu_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_rcu, NULL); diff --git a/sys/dev/drm/linux_scatterlist.c b/sys/dev/drm/linux_scatterlist.c index 92603464d7..8f64a8b36b 100644 --- a/sys/dev/drm/linux_scatterlist.c +++ b/sys/dev/drm/linux_scatterlist.c @@ -161,6 +161,7 @@ sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, struct page *page; char *vaddr; + kprintf("sg_pcopy_from_buffer\n"); off = 0; for_each_sg_page(sgl, &iter, nents, 0) { sg = iter.sg; @@ -177,10 +178,10 @@ sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, } len = min(curlen, buflen - off); page = sg_page_iter_page(&iter); - vaddr = (char *)kmap(page) + sg->offset; + vaddr = (char *)kmap_atomic_quick(page, pgprot_writecombine(0)) + sg->offset; memcpy(vaddr, (const char *)buf + off, len); off += len; - kunmap(page); + kunmap_atomic_quick(); } return (off); @@ -213,10 +214,10 @@ sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, } len = min(curlen, buflen - off); page = sg_page_iter_page(&iter); - vaddr = (char *)kmap(page) + sg->offset; + vaddr = (char *)kmap_atomic_quick(page, pgprot_writecombine(0)) + sg->offset; memcpy((char *)buf + off, vaddr, len); off += len; - kunmap(page); + kunmap_atomic_quick(); } return (off); diff --git a/sys/dev/drm/linux_shmem.c b/sys/dev/drm/linux_shmem.c index 8f9d8fb03d..dc702b9b69 100644 --- a/sys/dev/drm/linux_shmem.c +++ b/sys/dev/drm/linux_shmem.c @@ -38,6 +38,8 @@ #include #include +#include + /* * This code is typically called with a normal VM object to access * data from a userspace shared memory mapping. However, handle the @@ -78,6 +80,7 @@ shmem_read_mapping_page(vm_object_t object, vm_pindex_t pindex) m->dirty = 0; } } + drm_clflush_pages((struct page **)&m, 1); } vm_page_wire(m); /* put_page() undoes this */ vm_page_wakeup(m); diff --git a/sys/dev/drm/linux_vmalloc.c b/sys/dev/drm/linux_vmalloc.c index e07f8ea771..16bf81b5bd 100644 --- a/sys/dev/drm/linux_vmalloc.c +++ b/sys/dev/drm/linux_vmalloc.c @@ -31,6 +31,8 @@ #include #include +#include + struct vmap { void *addr; int npages; @@ -41,7 +43,26 @@ struct lock vmap_lock = LOCK_INITIALIZER("dlvml", 0, LK_CANRECURSE); SLIST_HEAD(vmap_list_head, vmap) vmap_list = SLIST_HEAD_INITIALIZER(vmap_list); -/* vmap: map an array of pages into virtually contiguous space */ +#if 0 +static inline int +place_to_memattr(int placement) +{ + if (placement & TTM_PL_FLAG_CACHED) + return VM_MEMATTR_WRITE_BACK; + if (placement & TTM_PL_FLAG_WC) + return VM_MEMATTR_WRITE_COMBINING; + if (placement & TTM_PL_FLAG_UNCACHED) + return VM_MEMATTR_UNCACHEABLE; + return VM_MEMATTR_WRITE_BACK; +} +#endif + +/* + * vmap: map an array of pages into virtually contiguous space + * + * Passed protection bits may be different from inherent page bits. + * prot is passed as linux _PAGE_* flags. + */ void * vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) @@ -60,7 +81,11 @@ vmap(struct page **pages, unsigned int count, vmp->addr = (void *)off; vmp->npages = count; - pmap_qenter(off, (struct vm_page **)pages, count); + pmap_qenter_memattr(off, (struct vm_page **)pages, count, + //VM_MEMATTR_UNCACHEABLE); + //place_to_memattr(prot)); /* YYY */ + pgflags_to_memattr(prot)); /* YYY */ + lockmgr(&vmap_lock, LK_EXCLUSIVE); SLIST_INSERT_HEAD(&vmap_list, vmp, vm_vmaps); lockmgr(&vmap_lock, LK_RELEASE); @@ -135,9 +160,6 @@ vfree(const void *addr) void * kvmalloc_array(size_t n, size_t size, gfp_t flags) { - if (n == 0) - return NULL; - if (n > SIZE_MAX / size) return NULL; diff --git a/sys/dev/drm/linux_wwmutex.c b/sys/dev/drm/linux_wwmutex.c index 0609420eb8..973b65c405 100644 --- a/sys/dev/drm/linux_wwmutex.c +++ b/sys/dev/drm/linux_wwmutex.c @@ -88,6 +88,17 @@ __wwlock(struct ww_mutex *ww, struct ww_acquire_ctx *ctx, if (intr) flags |= LK_PCATCH; + /* + * Already held by calling thread (DRM really needs this to work + * properly by returning -EALREADY) + */ + if (lockstatus(&ww->base, curthread) == LK_EXCLUSIVE) { + //kprintf("ww %p already locked\n", ww); + //print_backtrace(-1); + //tsleep(&flags, 0, "XXX", hz); + return -EALREADY; + } + /* * Normal mutex if ctx is NULL */ @@ -160,6 +171,14 @@ __wwlock(struct ww_mutex *ww, struct ww_acquire_ctx *ctx, } } +int +ww_mutex_lock_recursive(struct ww_mutex *ww) +{ + lockmgr(&ww->base, LK_EXCLUSIVE | LK_CANRECURSE); + + return 0; +} + int ww_mutex_lock(struct ww_mutex *ww, struct ww_acquire_ctx *ctx) { diff --git a/sys/dev/drm/scheduler/sched_entity.c b/sys/dev/drm/scheduler/sched_entity.c index 48463567cc..7eabf2faae 100644 --- a/sys/dev/drm/scheduler/sched_entity.c +++ b/sys/dev/drm/scheduler/sched_entity.c @@ -52,12 +52,12 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, { int i; - if (!(entity && rq_list && num_rq_list > 0 && rq_list[0])) + if (!(entity && rq_list && (num_rq_list == 0 || rq_list[0]))) return -EINVAL; memset(entity, 0, sizeof(struct drm_sched_entity)); INIT_LIST_HEAD(&entity->list); - entity->rq = rq_list[0]; + entity->rq = NULL; entity->guilty = guilty; entity->num_rq_list = num_rq_list; entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *), @@ -67,6 +67,9 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, for (i = 0; i < num_rq_list; ++i) entity->rq_list[i] = rq_list[i]; + + if (num_rq_list) + entity->rq = rq_list[0]; entity->last_scheduled = NULL; lockinit(&entity->rq_lock, "gserql", 0, LK_CANRECURSE); @@ -160,12 +163,15 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) #endif long ret = timeout; + if (!entity->rq) + return 0; + sched = entity->rq->sched; /** * The client will not queue more IBs during this fini, consume existing * queued IBs or discard them on SIGKILL */ - if (current->dfly_td->td_flags & TDF_EXITING) { + if (curproc && (curproc->p_flags & P_WEXIT)) { if (timeout) ret = wait_event_timeout( sched->job_scheduled, @@ -181,7 +187,7 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); #endif if (/*(!last_user || last_user == current->group_leader) && */ - (current->dfly_td->td_flags & TDF_EXITING) && fatal_signal_pending(current)) { + (curproc && (curproc->p_flags & P_WEXIT)) && fatal_signal_pending(current)) { lockmgr(&entity->rq_lock, LK_EXCLUSIVE); entity->stopped = true; drm_sched_rq_remove_entity(entity->rq, entity); @@ -262,20 +268,24 @@ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) */ void drm_sched_entity_fini(struct drm_sched_entity *entity) { - struct drm_gpu_scheduler *sched; + struct drm_gpu_scheduler *sched = NULL; - sched = entity->rq->sched; - drm_sched_rq_remove_entity(entity->rq, entity); + if (entity->rq) { + sched = entity->rq->sched; + drm_sched_rq_remove_entity(entity->rq, entity); + } /* Consumption of existing IBs wasn't completed. Forcefully * remove them here. */ if (spsc_queue_peek(&entity->job_queue)) { - /* Park the kernel for a moment to make sure it isn't processing - * our enity. - */ - kthread_park(sched->thread); - kthread_unpark(sched->thread); + if (sched) { + /* Park the kernel for a moment to make sure it isn't processing + * our enity. + */ + kthread_park(sched->thread); + kthread_unpark(sched->thread); + } if (entity->dependency) { dma_fence_remove_callback(entity->dependency, &entity->cb); @@ -360,9 +370,11 @@ void drm_sched_entity_set_priority(struct drm_sched_entity *entity, for (i = 0; i < entity->num_rq_list; ++i) drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority); - drm_sched_rq_remove_entity(entity->rq, entity); - drm_sched_entity_set_rq_priority(&entity->rq, priority); - drm_sched_rq_add_entity(entity->rq, entity); + if (entity->rq) { + drm_sched_rq_remove_entity(entity->rq, entity); + drm_sched_entity_set_rq_priority(&entity->rq, priority); + drm_sched_rq_add_entity(entity->rq, entity); + } lockmgr(&entity->rq_lock, LK_RELEASE); } diff --git a/sys/dev/drm/ttm/ttm_bo.c b/sys/dev/drm/ttm/ttm_bo.c index 012732b62c..844d4bc176 100644 --- a/sys/dev/drm/ttm/ttm_bo.c +++ b/sys/dev/drm/ttm/ttm_bo.c @@ -171,8 +171,10 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { #ifdef __DragonFly__ - if (WARN_ON(!list_empty(&bo->lru))) + if (WARN_ON(!list_empty(&bo->lru))) { + print_backtrace(-1); return; + } #endif man = &bdev->man[bo->mem.mem_type]; @@ -910,6 +912,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; + kprintf("ttm_bo_force_space: ttm_mem_evict_first\n"); ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); if (unlikely(ret != 0)) return ret; @@ -1404,6 +1407,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { lockmgr(&glob->lru_lock, LK_RELEASE); + kprintf("ttm_bo_force_list_clean: ttm_mem_evict_first\n"); ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; @@ -1800,11 +1804,12 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) ttm_bo_del_from_lru(bo); lockmgr(&glob->lru_lock, LK_RELEASE); - /** + /* * Move to system cached + * + * Apply https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=db9c1734ad69c0ba5e5e420ba31ebc1048976be6 */ - - if (bo->mem.mem_type != TTM_PL_SYSTEM || + if (bo->mem.mem_type != TTM_PL_SYSTEM || /* ZZZ */ bo->ttm->caching_state != tt_cached) { struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_reg evict_mem; @@ -1812,6 +1817,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) evict_mem = bo->mem; evict_mem.mm_node = NULL; evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; + //evict_mem.placement = TTM_PL_MASK_CACHING; / *ZZZ */ evict_mem.mem_type = TTM_PL_SYSTEM; ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); @@ -1869,6 +1875,8 @@ EXPORT_SYMBOL(ttm_bo_swapout_all); * unreserved * * @bo: Pointer to buffer + * + * YYY - make it uninterruptable, called from ttm_bo_vm.c w/return value ignored */ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) { @@ -1881,12 +1889,21 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) * bo::wu_mutex can go away if we change locking order to * mmap_sem -> bo::reserve. */ +#if 1 + mutex_lock(&bo->wu_mutex); /* YYY make uninterruptible */ + ret = 0; +#else ret = mutex_lock_interruptible(&bo->wu_mutex); +#endif if (unlikely(ret != 0)) return -ERESTARTSYS; if (!ww_mutex_is_locked(&bo->resv->lock)) goto out_unlock; +#if 1 + ret = reservation_object_lock(bo->resv, NULL); /* YYY make uninterruptible */ +#else ret = reservation_object_lock_interruptible(bo->resv, NULL); +#endif if (ret == -EINTR) ret = -ERESTARTSYS; if (unlikely(ret != 0)) diff --git a/sys/dev/drm/ttm/ttm_bo_util.c b/sys/dev/drm/ttm/ttm_bo_util.c index d24a91039e..f19b44ed00 100644 --- a/sys/dev/drm/ttm/ttm_bo_util.c +++ b/sys/dev/drm/ttm/ttm_bo_util.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -62,7 +63,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) - pr_err("Failed to expire sync object before unbinding TTM\n"); + pr_err("Failed to expire sync object before unbinding TTM ret = %d\n", ret); return ret; } @@ -262,7 +263,7 @@ static int ttm_copy_io_page(void *dst, void *src, unsigned long page) static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, unsigned long page, - pgprot_t prot) + pgprot_t prot) /* pgflags */ { struct page *d = ttm->pages[page]; void *dst; @@ -273,7 +274,10 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); #ifdef CONFIG_X86 - dst = kmap_atomic_prot(d, prot); + dst = kmap_atomic_quick(d, prot); + //dst = kmap_atomic_quick(d, pgprot_val(PAGE_KERNEL)); /* YYY */ + //drm_clflush_virt_range(src, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dst, PAGE_SIZE); /* YYY */ #else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) dst = vmap(&d, 1, 0, prot); @@ -283,10 +287,14 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, if (!dst) return -ENOMEM; + //drm_clflush_virt_range(src, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dst, PAGE_SIZE); /* YYY */ memcpy_fromio(dst, src, PAGE_SIZE); + //drm_clflush_virt_range(src, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dst, PAGE_SIZE); /* YYY */ #ifdef CONFIG_X86 - kunmap_atomic(dst); + kunmap_atomic_quick(); #else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) vunmap(dst); @@ -299,7 +307,7 @@ static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, unsigned long page, - pgprot_t prot) + pgprot_t prot) /* prot is pgflasg */ { struct page *s = ttm->pages[page]; void *src; @@ -309,7 +317,8 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); #ifdef CONFIG_X86 - src = kmap_atomic_prot(s, prot); + src = kmap_atomic_quick(s, prot); + //src = kmap_atomic_quick(s, pgprot_val(PAGE_KERNEL)); /* YYY */ #else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) src = vmap(&s, 1, 0, prot); @@ -319,10 +328,14 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, if (!src) return -ENOMEM; + //drm_clflush_virt_range(src, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dst, PAGE_SIZE); /* YYY */ memcpy_toio(dst, src, PAGE_SIZE); + //drm_clflush_virt_range(src, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dst, PAGE_SIZE); /* YYY */ #ifdef CONFIG_X86 - kunmap_atomic(src); + kunmap_atomic_quick(); #else if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) vunmap(src); diff --git a/sys/dev/drm/ttm/ttm_bo_vm.c b/sys/dev/drm/ttm/ttm_bo_vm.c index 4d9f642b47..6f676d89ba 100644 --- a/sys/dev/drm/ttm/ttm_bo_vm.c +++ b/sys/dev/drm/ttm/ttm_bo_vm.c @@ -48,6 +48,32 @@ #define TTM_BO_VM_NUM_PREFAULT 16 +static inline int +cstate_to_memattr(int cstate) +{ + switch(cstate) { + case tt_cached: + return VM_MEMATTR_WRITE_BACK; + case tt_wc: + return VM_MEMATTR_WRITE_COMBINING; + case tt_uncached: + return VM_MEMATTR_UNCACHEABLE; + } + return VM_MEMATTR_WRITE_BACK; +} + +static inline int +place_to_memattr(int placement) +{ + if (placement & TTM_PL_FLAG_CACHED) + return VM_MEMATTR_WRITE_BACK; + if (placement & TTM_PL_FLAG_WC) + return VM_MEMATTR_WRITE_COMBINING; + if (placement & TTM_PL_FLAG_UNCACHED) + return VM_MEMATTR_UNCACHEABLE; + return VM_MEMATTR_WRITE_BACK; +} + static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, struct vm_fault *vmf) { @@ -549,18 +575,25 @@ ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset, * for reserve, and if it fails, retry the fault after waiting * for the buffer to become unreserved. */ +#if 1 + /* use blocking, uninterruptable */ + ret = ttm_bo_reserve(bo, false, false, NULL); /* YYY */ +#else ret = ttm_bo_reserve(bo, true, true, NULL); +#endif if (unlikely(ret != 0)) { if (ret != -EBUSY) { retval = VM_PAGER_ERROR; goto out_unlock2; } - if (vmf->flags & FAULT_FLAG_ALLOW_RETRY || 1) { + if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) || 1) { if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + ttm_bo_get(bo); up_read(&vma->vm_mm->mmap_sem); - (void) ttm_bo_wait_unreserved(bo); + (void) ttm_bo_wait_unreserved(bo); /* YYY assumed uninterruptible */ down_read(&vma->vm_mm->mmap_sem); + ttm_bo_put(bo); } #ifndef __DragonFly__ @@ -619,7 +652,11 @@ ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset, } } +#if 1 + ret = ttm_mem_io_lock(man, false); /* YYY make uninterruptable */ +#else ret = ttm_mem_io_lock(man, true); +#endif if (unlikely(ret != 0)) { retval = VM_PAGER_ERROR; goto out_unlock1; @@ -666,7 +703,6 @@ ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset, #ifdef __DragonFly__ m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base + bo->mem.bus.offset + offset); - pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement, 0)); #endif cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, cvma.vm_page_prot); @@ -692,9 +728,6 @@ ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset, retval = VM_PAGER_ERROR; goto out_io_unlock1; } - pmap_page_set_memattr(m, - (bo->mem.placement & TTM_PL_FLAG_CACHED) ? - VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement, 0)); } if (vm_page_busy_try(m, FALSE)) { @@ -706,6 +739,14 @@ ttm_bo_vm_fault_dfly(vm_object_t vm_obj, vm_ooffset_t offset, goto retry; } +#if 1 + /* YYY set memattr according to BO, TTM might be NULL */ + pmap_page_set_memattr(m, place_to_memattr(bo->mem.placement)); +#else + pmap_page_set_memattr(m, VM_MEMATTR_UNCACHEABLE); +#endif +//kprintf("PF: bo %p ttm %p vmo %p off %ld vm_page %p phys_addr 0x%lx\n", bo, ttm, vm_obj, offset, m, VM_PAGE_TO_PHYS(m)); + /* * Return our fake page BUSYd. Do not index it into the VM object. * The caller will enter it into the pmap. @@ -794,7 +835,7 @@ ttm_bo_mmap_single(struct file *fp, struct drm_device *dev, * setup our own VM object and ignore what the linux code did other * then supplying us the 'bo'. */ - ret = ttm_bo_mmap(fp, &vma, bdev); + ret = ttm_bo_mmap(NULL, &vma, bdev); if (ret == 0) { bo = vma.vm_private_data; @@ -802,6 +843,7 @@ ttm_bo_mmap_single(struct file *fp, struct drm_device *dev, &ttm_pager_ops, size, nprot, 0, curthread->td_ucred); + vm_obj->memattr = VM_MEMATTR_WRITE_BACK; if (vm_obj) { *obj_res = vm_obj; *offset = 0; /* object-relative offset */ diff --git a/sys/dev/drm/ttm/ttm_execbuf_util.c b/sys/dev/drm/ttm/ttm_execbuf_util.c index 57ea023252..75baddd0eb 100644 --- a/sys/dev/drm/ttm/ttm_execbuf_util.c +++ b/sys/dev/drm/ttm/ttm_execbuf_util.c @@ -196,14 +196,34 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, driver = bdev->driver; glob = bo->bdev->glob; + list_for_each_entry(entry, list, head) { + bo = entry->bo; + if (!list_empty(&bo->lru)) + kprintf("a<%p>", bo); + } + lockmgr(&glob->lru_lock, LK_EXCLUSIVE); list_for_each_entry(entry, list, head) { bo = entry->bo; + if (!list_empty(&bo->lru)) { + struct ttm_validate_buffer *entry2; + + kprintf("b<%p:%d>: ", bo, !!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)); + list_for_each_entry(entry2, list, head) { + kprintf("%p[%d:%d] ", + entry2->bo, + !list_empty(&entry2->bo->lru), + !!(entry2->bo->mem.placement & TTM_PL_FLAG_NO_EVICT) + ); + } + kprintf("\n"); + } if (entry->shared) reservation_object_add_shared_fence(bo->resv, fence); else reservation_object_add_excl_fence(bo->resv, fence); + ttm_bo_add_to_lru(bo); reservation_object_unlock(bo->resv); } diff --git a/sys/dev/drm/ttm/ttm_memory.c b/sys/dev/drm/ttm/ttm_memory.c index d0d92bcc73..536bd361b8 100644 --- a/sys/dev/drm/ttm/ttm_memory.c +++ b/sys/dev/drm/ttm/ttm_memory.c @@ -197,8 +197,11 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, target = (extra > target) ? 0ULL : target; - if (zone->used_mem > target) + if (zone->used_mem > target) { + kprintf("ttm_zones_above_swap_target: glob %p from_wq=%d i=%d extra=%ld, zone=%p, used_mem=%ld target=%ld\n", + glob, from_wq, i, extra, zone, zone->used_mem, target); return true; + } } return false; } @@ -215,10 +218,15 @@ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, { int ret; +//kprintf("ttm_shrink\n"); +// return; + lockmgr(&glob->lock, LK_EXCLUSIVE); while (ttm_zones_above_swap_target(glob, from_wq, extra)) { lockmgr(&glob->lock, LK_RELEASE); + kprintf("ttm_shrink: zones above swap target, extra %ld\n", + extra); ret = ttm_bo_swapout(glob->bo_glob, ctx); lockmgr(&glob->lock, LK_EXCLUSIVE); if (unlikely(ret != 0)) @@ -333,6 +341,10 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) */ mem = (uint64_t)vm_contig_avail_pages() * PAGE_SIZE; +kprintf("vm_contig_avail_pages()=%ld mem=%ld\n", vm_contig_avail_pages(), mem); + +//mem=1073741824; + ret = ttm_mem_init_kernel_zone(glob, mem); if (unlikely(ret != 0)) goto out_no_zone; diff --git a/sys/dev/drm/ttm/ttm_page_alloc.c b/sys/dev/drm/ttm/ttm_page_alloc.c index c90f3bdc28..3d592db54c 100644 --- a/sys/dev/drm/ttm/ttm_page_alloc.c +++ b/sys/dev/drm/ttm/ttm_page_alloc.c @@ -55,6 +55,8 @@ #include #include +#include + #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) #define SMALL_ALLOCATION 16 #define FREE_ALL_PAGES (~0U) @@ -453,10 +455,11 @@ static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); } -static int ttm_set_pages_caching(struct page **pages, +int ttm_set_pages_caching(struct page **pages, enum ttm_caching_state cstate, unsigned cpages) { int r = 0; + /* Set page caching */ switch (cstate) { case tt_uncached: @@ -470,6 +473,9 @@ static int ttm_set_pages_caching(struct page **pages, pr_err("Failed to set %d pages to wc!\n", cpages); break; default: + r = ttm_set_pages_array_wb(pages, cpages); + if (r) + pr_err("Failed to set %d pages to wb!\n", cpages); break; } return r; @@ -509,6 +515,7 @@ static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags, unsigned npages = 1 << order; unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); +//kprintf("alloc_new_pages: cstate %d count %u order %u npages %u max_cpages %u\n", cstate, count, order, npages, max_cpages); /* allocate array for page caching change */ caching_array = kmalloc(max_cpages*sizeof(struct page *), M_DRM, M_WAITOK); @@ -552,7 +559,7 @@ static int ttm_alloc_new_pages(struct pglist *pages, gfp_t gfp_flags, if (cpages == max_cpages) { r = ttm_set_pages_caching(caching_array, - cstate, cpages); + cstate, cpages); if (r) { ttm_handle_caching_state_failure(pages, ttm_flags, cstate, @@ -670,6 +677,7 @@ static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, count = 0; out: spin_unlock_irqrestore(&pool->lock, irq_flags); + return count; } @@ -754,6 +762,8 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, } pages[r] = (struct page *)p; } + /* YYY pages from above may not have the correct caching mode */ + ttm_set_pages_caching(pages, cstate, npages); return 0; } @@ -768,12 +778,16 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, pages[count++] = (struct page *)p; } + /* YYY pages from above may not have the correct caching mode */ + ttm_set_pages_caching(pages, cstate, count); + /* clear the pages coming from the pool if requested */ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { TAILQ_FOREACH(p, &plist, pageq) { pmap_zero_page(VM_PAGE_TO_PHYS(p)); } } + drm_clflush_pages(pages, count); /* If pool didn't have enough pages allocate new one. */ if (npages > 0) { @@ -899,6 +913,8 @@ int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) return -ENOMEM; + //kprintf("ttm_pool_populate: Get %ld pages cstate %d\n", + // ttm->num_pages, ttm->caching_state); ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, ttm->caching_state); if (unlikely(ret != 0)) { @@ -937,7 +953,10 @@ EXPORT_SYMBOL(ttm_pool_unpopulate); int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, struct ttm_operation_ctx *ctx) { - unsigned i, j; + unsigned i; +#if 0 + unsigned j; +#endif int r; r = ttm_pool_populate(&tt->ttm, ctx); @@ -945,8 +964,9 @@ int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, return r; for (i = 0; i < tt->ttm.num_pages; ++i) { - struct page *p = tt->ttm.pages[i]; size_t num_pages = 1; +#if 0 + struct page *p = tt->ttm.pages[i]; for (j = i + 1; j < tt->ttm.num_pages; ++j) { if (++p != tt->ttm.pages[j]) @@ -954,10 +974,14 @@ int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, ++num_pages; } +#endif tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i], 0, num_pages * PAGE_SIZE, DMA_BIDIRECTIONAL); + +//kprintf("map_pages: ttm %p num_pages %ld dma_address[%d] 0x%lx\n", &tt->ttm, tt->ttm.num_pages, i, tt->dma_address[i]); +#if 0 if (dma_mapping_error(dev, tt->dma_address[i])) { while (i--) { dma_unmap_page(dev, tt->dma_address[i], @@ -972,7 +996,9 @@ int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt, tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; ++i; } +#endif } +//kprintf("map_pages: ttm %p num_pages %ld dma_address 0x%lx-0x%lx\n", &tt->ttm, tt->ttm.num_pages, tt->dma_address[0], tt->dma_address[tt->ttm.num_pages-1]); return 0; } EXPORT_SYMBOL(ttm_populate_and_map_pages); diff --git a/sys/dev/drm/ttm/ttm_page_alloc_dma.c b/sys/dev/drm/ttm/ttm_page_alloc_dma.c index d594f7520b..82c8ff139e 100644 --- a/sys/dev/drm/ttm/ttm_page_alloc_dma.c +++ b/sys/dev/drm/ttm/ttm_page_alloc_dma.c @@ -273,12 +273,17 @@ static int ttm_set_pages_caching(struct dma_pool *pool, if (r) pr_err("%s: Failed to set %d pages to uc!\n", pool->dev_name, cpages); - } + } else if (pool->type & IS_WC) { r = ttm_set_pages_array_wc(pages, cpages); if (r) pr_err("%s: Failed to set %d pages to wc!\n", pool->dev_name, cpages); + } else { + r = ttm_set_pages_array_wb(pages, cpages); + if (r) + pr_err("%s: Failed to set %d pages to wb!\n", + pool->dev_name, cpages); } return r; } diff --git a/sys/dev/drm/ttm/ttm_tt.c b/sys/dev/drm/ttm/ttm_tt.c index cb888b791b..a7cd86f7dd 100644 --- a/sys/dev/drm/ttm/ttm_tt.c +++ b/sys/dev/drm/ttm/ttm_tt.c @@ -43,7 +43,7 @@ #include #include #include -#include +#include /* ZZZ */ /** * Allocates a ttm structure for the given BO. @@ -118,6 +118,12 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm) return 0; } +/* + * Remove + * + * https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=db9c1734ad69c0ba5e5e420ba31ebc1048976be6 + */ +#if 1 /* ZZZ */ static int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_old, enum ttm_caching_state c_new) @@ -129,6 +135,8 @@ static int ttm_tt_set_page_caching(struct page *p, return 0; #endif + //kprintf("tt_set_page_caching 0x%lx c_old %d c_new %d\n", PHYS_TO_DMAP(VM_PAGE_TO_PHYS((struct vm_page*)p)), c_old, c_new); + if (c_old != tt_cached) { /* p isn't in the default caching state, set it to * writeback first to free its current memtype. */ @@ -138,19 +146,48 @@ static int ttm_tt_set_page_caching(struct page *p, return ret; } + // if (c_new == tt_wc) + // pmap_page_set_memattr((struct vm_page *)p, VM_MEMATTR_WRITE_COMBINING); if (c_new == tt_wc) - pmap_page_set_memattr((struct vm_page *)p, VM_MEMATTR_WRITE_COMBINING); + ret = ttm_set_pages_wc(p, 1); else if (c_new == tt_uncached) ret = ttm_set_pages_uc(p, 1); return ret; } +#endif /* * Change caching policy for the linear kernel map * for range of pages in a ttm. + * + * https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=db9c1734ad69c0ba5e5e420ba31ebc1048976be6 */ +#if 0 /* ZZZ */ +/* new version */ +static int ttm_tt_set_caching(struct ttm_tt *ttm, + enum ttm_caching_state c_state) +{ + if (ttm->caching_state != c_state) { + /* Can't change the caching state after TT is populated */ + if (WARN_ON_ONCE((ttm->state != tt_unpopulated))) { + kprintf("ttm_tt_set_caching state %d\n", ttm->state); + print_backtrace(-1); + return -EINVAL; + } + + ttm->caching_state = c_state; + } + /* YYY underlying pages may not have synchronized cache states */ + if (ttm->state != tt_unpopulated) + ttm_set_pages_caching(ttm->pages, c_state, ttm->num_pages); + return 0; +} + +#else +/* old version */ + static int ttm_tt_set_caching(struct ttm_tt *ttm, enum ttm_caching_state c_state) { @@ -167,8 +204,18 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, return 0; } + /* + * Unconditionally clflush because speculative reads can cache + * pages from uncacheable memory and then become stale when we + * make the memory cacheable again, if DMA modified the real + * memory after the speculative read. + */ +#if 1 + drm_clflush_pages(ttm->pages, ttm->num_pages); +#else if (ttm->caching_state == tt_cached) drm_clflush_pages(ttm->pages, ttm->num_pages); +#endif for (i = 0; i < ttm->num_pages; ++i) { cur_page = ttm->pages[i]; @@ -196,11 +243,33 @@ static int ttm_tt_set_caching(struct ttm_tt *ttm, return ret; } +#endif int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) { enum ttm_caching_state state; + /* + * Don't change the ttm caching state unless we have to + */ + switch (ttm->caching_state) { + case tt_wc: + if (placement & TTM_PL_FLAG_WC) + placement = TTM_PL_FLAG_WC; + break; + case tt_uncached: + if (placement & TTM_PL_FLAG_UNCACHED) + placement = TTM_PL_FLAG_UNCACHED; + break; + case tt_cached: + if (placement & TTM_PL_FLAG_CACHED) + placement = TTM_PL_FLAG_CACHED; + break; + } + + /* + * Generally desired state from placement + */ if (placement & TTM_PL_FLAG_WC) state = tt_wc; else if (placement & TTM_PL_FLAG_UNCACHED) @@ -355,8 +424,6 @@ EXPORT_SYMBOL(ttm_tt_bind); int ttm_tt_swapin(struct ttm_tt *ttm) { vm_object_t swap_storage; - struct page *from_page; - struct page *to_page; int i; int ret = -ENOMEM; @@ -366,29 +433,46 @@ int ttm_tt_swapin(struct ttm_tt *ttm) VM_OBJECT_LOCK(swap_storage); vm_object_pip_add(swap_storage, 1); for (i = 0; i < ttm->num_pages; ++i) { - from_page = (struct page *)vm_page_grab(swap_storage, i, VM_ALLOC_NORMAL | - VM_ALLOC_RETRY); - if (((struct vm_page *)from_page)->valid != VM_PAGE_BITS_ALL) { + vm_page_t from_page; + struct page *to_page; + void *kptr; + void *sptr; + + from_page = vm_page_grab(swap_storage, i, + VM_ALLOC_NORMAL | VM_ALLOC_RETRY); + if (from_page->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(swap_storage, i)) { if (vm_pager_get_page(swap_storage, i, - (struct vm_page **)&from_page, 1) != VM_PAGER_OK) { - vm_page_free((struct vm_page *)from_page); + &from_page, 1) != VM_PAGER_OK) + { + vm_page_free(from_page); ret = -EIO; goto out_err; } } else { - vm_page_zero_invalid((struct vm_page *)from_page, TRUE); + vm_page_zero_invalid(from_page, TRUE); } } + to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) { vm_page_wakeup((struct vm_page *)from_page); goto out_err; } - pmap_copy_page(VM_PAGE_TO_PHYS((struct vm_page *)from_page), - VM_PAGE_TO_PHYS((struct vm_page *)to_page)); - vm_page_wakeup((struct vm_page *)from_page); + sptr = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(from_page)); + kptr = kmap_atomic(to_page); + //drm_clflush_virt_range(sptr, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(kptr, PAGE_SIZE); /* YYY */ + bcopy(sptr, kptr, PAGE_SIZE); + //drm_clflush_virt_range(sptr, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(kptr, PAGE_SIZE); /* YYY */ + //pmap_copy_page(VM_PAGE_TO_PHYS((struct vm_page *)from_page), + // VM_PAGE_TO_PHYS((struct vm_page *)to_page)); + vm_page_wakeup(from_page); + drm_clflush_virt_range(kptr, PAGE_SIZE); + //drm_clflush_pages(&to_page, 1); + kunmap_atomic(kptr); } vm_object_pip_wakeup(swap_storage); VM_OBJECT_UNLOCK(swap_storage); @@ -409,11 +493,12 @@ int ttm_tt_swapin(struct ttm_tt *ttm) int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) { vm_object_t obj; - vm_page_t from_page, to_page; + struct page *from_page; + vm_page_t to_page; int i; BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); - BUG_ON(ttm->caching_state != tt_cached); + //BUG_ON(ttm->caching_state != tt_cached); if (!persistent_swap_storage) { obj = swap_pager_alloc(NULL, @@ -422,22 +507,35 @@ int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) pr_err("Failed allocating swap storage\n"); return (-ENOMEM); } - } else + } else { obj = persistent_swap_storage; + } VM_OBJECT_LOCK(obj); vm_object_pip_add(obj, 1); for (i = 0; i < ttm->num_pages; ++i) { - from_page = (struct vm_page *)ttm->pages[i]; + void *kptr; + void *dptr; + + from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); - pmap_copy_page(VM_PAGE_TO_PHYS(from_page), - VM_PAGE_TO_PHYS(to_page)); + kptr = kmap_atomic(from_page); + dptr = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(to_page)); + //drm_clflush_virt_range(kptr, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dptr, PAGE_SIZE); /* YYY */ + bcopy(kptr, dptr, PAGE_SIZE); + //drm_clflush_virt_range(kptr, PAGE_SIZE); /* YYY */ + //drm_clflush_virt_range(dptr, PAGE_SIZE); /* YYY */ + //pmap_copy_page(VM_PAGE_TO_PHYS(from_page), + // VM_PAGE_TO_PHYS(to_page)); to_page->valid = VM_PAGE_BITS_ALL; vm_page_dirty(to_page); vm_page_wakeup(to_page); + drm_clflush_virt_range(kptr, PAGE_SIZE); + kunmap_atomic(kptr); } vm_object_pip_wakeup(obj); VM_OBJECT_UNLOCK(obj); diff --git a/sys/dev/sound/pci/hda/hdac.c b/sys/dev/sound/pci/hda/hdac.c index 14a54fe1b1..8342fb2871 100644 --- a/sys/dev/sound/pci/hda/hdac.c +++ b/sys/dev/sound/pci/hda/hdac.c @@ -241,7 +241,6 @@ TASKQUEUE_DEFINE_THREAD(hdac); * Function prototypes ****************************************************************************/ static void hdac_intr_handler(void *); -static void hdac_one_intr(struct hdac_softc *sc, uint32_t intsts); static int hdac_reset(struct hdac_softc *, int); static int hdac_get_capabilities(struct hdac_softc *); static void hdac_dma_cb(void *, bus_dma_segment_t *, int, int); @@ -339,35 +338,20 @@ static void hdac_intr_handler(void *context) { struct hdac_softc *sc; + device_t dev; uint32_t intsts; + uint8_t rirbsts; + int i; sc = (struct hdac_softc *)context; - - /* - * Loop until HDAC_INTSTS_GIS gets clear. - * It is plausible that hardware interrupts a host only when GIS goes - * from zero to one. GIS is formed by OR-ing multiple hardware - * statuses, so it's possible that a previously cleared status gets set - * again while another status has not been cleared yet. Thus, there - * will be no new interrupt as GIS always stayed set. If we don't - * re-examine GIS then we can leave it set and never get an interrupt - * again. - */ hdac_lock(sc); + + /* Do we have anything to do? */ intsts = HDAC_READ_4(&sc->mem, HDAC_INTSTS); - while (intsts != 0xffffffff && (intsts & HDAC_INTSTS_GIS) != 0) { - hdac_one_intr(sc, intsts); - intsts = HDAC_READ_4(&sc->mem, HDAC_INTSTS); + if ((intsts & HDAC_INTSTS_GIS) == 0) { + hdac_unlock(sc); + return; } - hdac_unlock(sc); -} - -static void -hdac_one_intr(struct hdac_softc *sc, uint32_t intsts) -{ - device_t dev; - uint8_t rirbsts; - int i; /* Was this a controller interrupt? */ if (intsts & HDAC_INTSTS_CIS) { @@ -395,7 +379,9 @@ hdac_one_intr(struct hdac_softc *sc, uint32_t intsts) } } } + HDAC_WRITE_4(&sc->mem, HDAC_INTSTS, intsts); + hdac_unlock(sc); } static void @@ -1281,6 +1267,9 @@ hdac_attach(device_t dev) /* Allocate resources */ result = hdac_mem_alloc(sc); + if (result != 0) + goto hdac_attach_fail; + result = hdac_irq_alloc(sc); if (result != 0) goto hdac_attach_fail; @@ -1353,10 +1342,6 @@ hdac_attach(device_t dev) hdac_corb_init(sc); hdac_rirb_init(sc); - result = hdac_irq_alloc(sc); - if (result != 0) - goto hdac_attach_fail; - /* Defer remaining of initialization until interrupts are enabled */ sc->intrhook.ich_func = hdac_attach2; sc->intrhook.ich_arg = (void *)sc; diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index dc8adfe23b..c96151c0fc 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -170,13 +170,19 @@ lockmgr_shared(struct lock *lkp, u_int flags) pflags = (extflags & LK_PCATCH) ? PCATCH : 0; timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; - tsleep_interlock(lkp, pflags); + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); count = atomic_fetchadd_long(&lkp->lk_count, 0); if ((count & LKC_SHARED) && (count & (LKC_EXREQ | LKC_UPREQ))) { - error = tsleep(lkp, pflags | PINTERLOCKED, - lkp->lk_wmesg, timo); + if (extflags & LK_SPIN) { + cpu_pause(); + error = 0; + } else { + error = tsleep(lkp, pflags | PINTERLOCKED, + lkp->lk_wmesg, timo); + } if (error) return error; count = lkp->lk_count; @@ -247,8 +253,13 @@ lockmgr_shared(struct lock *lkp, u_int flags) * Interlocked after the first loop. */ if (didloop) { - error = tsleep(lkp, pflags | PINTERLOCKED, - lkp->lk_wmesg, timo); + if ((extflags & LK_SPIN) == 0) { + error = tsleep(lkp, pflags | PINTERLOCKED, + lkp->lk_wmesg, timo); + } else { + cpu_pause(); + error = 0; + } if (extflags & LK_SLEEPFAIL) { undo_shreq(lkp); error = ENOLCK; @@ -268,7 +279,8 @@ lockmgr_shared(struct lock *lkp, u_int flags) count = lkp->lk_count; if (count & LKC_SHARED) break; - tsleep_interlock(lkp, pflags); + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); count = atomic_fetchadd_64(&lkp->lk_count, 0); } if (error == 0) @@ -366,7 +378,10 @@ lockmgr_exclusive(struct lock *lkp, u_int flags) ncount = count | LKC_EXREQ2; else ncount = count | LKC_EXREQ; - tsleep_interlock(lkp, pflags); + + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); + if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { /* * If we successfully transitioned to EXREQ we @@ -378,8 +393,13 @@ lockmgr_exclusive(struct lock *lkp, u_int flags) break; } - error = tsleep(lkp, pflags | PINTERLOCKED, - lkp->lk_wmesg, timo); + if ((extflags & LK_SPIN) == 0) { + error = tsleep(lkp, pflags | PINTERLOCKED, + lkp->lk_wmesg, timo); + } else { + cpu_pause(); + error = 0; + } count = lkp->lk_count; /* relod */ cpu_ccfence(); } @@ -437,7 +457,13 @@ lockmgr_exclusive(struct lock *lkp, u_int flags) pflags = (extflags & LK_PCATCH) ? PCATCH : 0; timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; - error = tsleep(lkp, pflags | PINTERLOCKED, lkp->lk_wmesg, timo); + if ((extflags & LK_SPIN) == 0) { + error = tsleep(lkp, pflags | PINTERLOCKED, + lkp->lk_wmesg, timo); + } else { + cpu_pause(); + error = 0; + } #ifdef INVARIANTS if (lock_test_mode > 0) { --lock_test_mode; @@ -482,7 +508,8 @@ lockmgr_exclusive(struct lock *lkp, u_int flags) COUNT(td, 1); break; } - tsleep_interlock(lkp, pflags); + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); count = atomic_fetchadd_64(&lkp->lk_count, 0); } return error; @@ -649,7 +676,8 @@ lockmgr_upgrade(struct lock *lkp, u_int flags) * upgrade and break to the next loop. */ pflags = (extflags & LK_PCATCH) ? PCATCH : 0; - tsleep_interlock(lkp, pflags); + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); ncount = (count - LKC_SCOUNT) | LKC_UPREQ; if (atomic_fcmpset_64(&lkp->lk_count, &count, ncount)) { count = ncount; @@ -696,7 +724,13 @@ lockmgr_upgrade(struct lock *lkp, u_int flags) pflags = (extflags & LK_PCATCH) ? PCATCH : 0; timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; - error = tsleep(lkp, pflags | PINTERLOCKED, lkp->lk_wmesg, timo); + if ((extflags & LK_SPIN) == 0) { + error = tsleep(lkp, pflags | PINTERLOCKED, + lkp->lk_wmesg, timo); + } else { + cpu_pause(); + error = 0; + } if (extflags & LK_SLEEPFAIL) { if (undo_upreq(lkp) == 0) { lkp->lk_lockholder = LK_KERNTHREAD; @@ -725,7 +759,8 @@ lockmgr_upgrade(struct lock *lkp, u_int flags) lkp->lk_lockholder = td; break; } - tsleep_interlock(lkp, pflags); + if ((extflags & LK_SPIN) == 0) + tsleep_interlock(lkp, pflags); count = atomic_fetchadd_64(&lkp->lk_count, 0); /* retry */ } diff --git a/sys/platform/pc64/include/globaldata.h b/sys/platform/pc64/include/globaldata.h index 2c5eb51bcc..d5070d2dc7 100644 --- a/sys/platform/pc64/include/globaldata.h +++ b/sys/platform/pc64/include/globaldata.h @@ -84,8 +84,8 @@ struct mdglobaldata { long gd_lunused1; long gd_lunused2; long gd_lunusde3; - caddr_t gd_aunused0; - caddr_t gd_aunused1; + register_t *gd_temp_pte; + vm_offset_t gd_temp_va; caddr_t gd_aunused2; struct pv_entry *gd_newpv; u_int gd_acpi_id; diff --git a/sys/platform/pc64/include/pmap.h b/sys/platform/pc64/include/pmap.h index c1aa114c88..0b9da4e7c2 100644 --- a/sys/platform/pc64/include/pmap.h +++ b/sys/platform/pc64/include/pmap.h @@ -378,6 +378,9 @@ void pmap_unmapdev (vm_offset_t, vm_size_t); struct vm_page *pmap_use_pt (pmap_t, vm_offset_t); void pmap_set_opt (void); void pmap_init_pat(void); +vm_offset_t pmap_tempmap_enter(vm_paddr_t pa, int memattr); +void pmap_tempmap_exit(void); +void pmap_invalidate_cache_page(vm_page_t m); void pmap_invalidate_cache_pages(vm_page_t *pages, int count); void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva); diff --git a/sys/platform/pc64/x86_64/busdma_machdep.c b/sys/platform/pc64/x86_64/busdma_machdep.c index 79a21ca93a..e733047a23 100644 --- a/sys/platform/pc64/x86_64/busdma_machdep.c +++ b/sys/platform/pc64/x86_64/busdma_machdep.c @@ -538,6 +538,9 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, if (*vaddr == NULL) return (ENOMEM); + /* + * NOTE: We don't use kmalloc if the attr is not the default + */ if (attr != VM_MEMATTR_DEFAULT) { pmap_change_attr((vm_offset_t)(*vaddr), dmat->maxsize / PAGE_SIZE, attr); @@ -554,14 +557,21 @@ bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { /* * dmamem does not need to be bounced, so the map should be - * NULL + * NULL. + * + * kmalloc()s always have a default memattr so we only have + * to restore the attribute when using contigfree(). */ if (map != NULL && map != (void *)-1) panic("bus_dmamem_free: Invalid map freed"); - if (map == NULL) + if (map == NULL) { kfree(vaddr, M_DEVBUF); - else + } else { + pmap_change_attr((vm_offset_t)vaddr, + dmat->maxsize / PAGE_SIZE, + VM_MEMATTR_DEFAULT); contigfree(vaddr, dmat->maxsize, M_DEVBUF); + } } static __inline vm_paddr_t diff --git a/sys/platform/pc64/x86_64/pmap.c b/sys/platform/pc64/x86_64/pmap.c index 7f5d9b9468..e52fb02845 100644 --- a/sys/platform/pc64/x86_64/pmap.c +++ b/sys/platform/pc64/x86_64/pmap.c @@ -239,7 +239,8 @@ __read_frequently static boolean_t pmap_initialized = FALSE; //static int pgeflag; /* PG_G or-in */ static uint64_t PatMsr; /* value of MSR_PAT */ -static int ndmpdp; +static int ndmpdp; /* count 1GB pgtable pages needed for DMAP */ +static int ndmpgs; /* count 2MB pgtable pages needed for DMAP */ static vm_paddr_t dmaplimit; vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; @@ -255,6 +256,7 @@ uint64_t KPML4phys; /* phys addr of kernel level 4 */ static uint64_t DMPDphys; /* phys addr of direct mapped level 2 */ static uint64_t DMPDPphys; /* phys addr of direct mapped level 3 */ +static uint64_t DMPGphys; /* phys addr of direct mapped level 4 */ /* * Data for the pv entry allocation mechanism @@ -335,6 +337,9 @@ static long vm_pmap_pv_entries; SYSCTL_LONG(_vm, OID_AUTO, pmap_pv_entries, CTLFLAG_RD, &vm_pmap_pv_entries, 0, ""); +SYSCTL_INT(_machdep, OID_AUTO, ndmpgs, CTLFLAG_RD, + &ndmpgs, 0, ""); + /* Standard user access funtions */ extern int std_copyinstr (const void *udaddr, void *kaddr, size_t len, size_t *lencopied); @@ -884,10 +889,10 @@ create_pagetables(vm_paddr_t *firstaddr) if (ndmpdp < 512) /* Minimum 512GB of DMAP */ ndmpdp = 512; #endif - KKASSERT(ndmpdp <= NDMPML4E * NPML4EPG); DMapMaxAddress = DMAP_MIN_ADDRESS + ((ndmpdp * NPDEPG) << PDRSHIFT); + ndmpgs = ndmpdp * NPDEPG; /* * Starting at KERNBASE - map all 2G worth of page table pages. @@ -951,12 +956,8 @@ create_pagetables(vm_paddr_t *firstaddr) * attributes on a fixed grain (see pmap_change_attr()). */ DMPDPphys = allocpages(firstaddr, NDMPML4E); -#if 1 - DMPDphys = allocpages(firstaddr, ndmpdp); -#else - if ((amd_feature & AMDID_PAGE1GB) == 0) - DMPDphys = allocpages(firstaddr, ndmpdp); -#endif + DMPDphys = allocpages(firstaddr, ndmpdp); /* 1GB w/2MB ptes */ + DMPGphys = allocpages(firstaddr, ndmpgs); /* 2MB w/4K ptes */ dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; /* @@ -1037,64 +1038,54 @@ create_pagetables(vm_paddr_t *firstaddr) } /* - * Now set up the direct map space using either 2MB or 1GB pages - * Preset PG_M and PG_A because demotion expects it. + * Now set up the direct map. Preset PG_M and PG_A because demotion + * expects it. * - * When filling in entries in the PD pages make sure any excess - * entries are set to zero as we allocated enough PD pages + * We generate a full-resolution DMAP because cache mode aliasing has + * become too big a problem, particularly aliasing caused by + * speculative reads, and we want to be able to use the DMAP. * - * Stuff for our DMAP. Use 2MB pages even when 1GB pages - * are available in order to allow APU code to adjust page - * attributes on a fixed grain (see pmap_change_attr()). + * So we create the full 1GB->2MB->4K table instead of stopping at + * 1GB or 2MB. */ -#if 0 - if ((amd_feature & AMDID_PAGE1GB) == 0) -#endif - { - /* - * Use 2MB pages - */ - for (i = 0; i < NPDEPG * ndmpdp; i++) { - ((pd_entry_t *)DMPDphys)[i] = i << PDRSHIFT; - ((pd_entry_t *)DMPDphys)[i] |= - pmap_bits_default[PG_RW_IDX] | - pmap_bits_default[PG_V_IDX] | - pmap_bits_default[PG_PS_IDX] | - pmap_bits_default[PG_G_IDX] | - pmap_bits_default[PG_M_IDX] | - pmap_bits_default[PG_A_IDX]; - } - /* - * And the direct map space's PDP - */ - for (i = 0; i < ndmpdp; i++) { - ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + - (i << PAGE_SHIFT); - ((pdp_entry_t *)DMPDPphys)[i] |= - pmap_bits_default[PG_RW_IDX] | - pmap_bits_default[PG_V_IDX] | - pmap_bits_default[PG_A_IDX]; - } + /* + * Populate 4K PTEs (PT pages), pointing to physical memory. + */ + for (i = 0; i < NPTEPG * ndmpgs; i++) { + ((pt_entry_t *)DMPGphys)[i] = i << PAGE_SHIFT; + ((pt_entry_t *)DMPGphys)[i] |= + pmap_bits_default[PG_RW_IDX] | + pmap_bits_default[PG_V_IDX] | + pmap_bits_default[PG_G_IDX] | + pmap_bits_default[PG_M_IDX] | + pmap_bits_default[PG_A_IDX]; } -#if 0 - else { - /* - * 1GB pages - */ - for (i = 0; i < ndmpdp; i++) { - ((pdp_entry_t *)DMPDPphys)[i] = - (vm_paddr_t)i << PDPSHIFT; - ((pdp_entry_t *)DMPDPphys)[i] |= - pmap_bits_default[PG_RW_IDX] | - pmap_bits_default[PG_V_IDX] | - pmap_bits_default[PG_PS_IDX] | - pmap_bits_default[PG_G_IDX] | - pmap_bits_default[PG_M_IDX] | - pmap_bits_default[PG_A_IDX]; - } + + /* + * Populate 2MB PTEs (PD pages), point to DMPG pages + */ + for (i = 0; i < NPDEPG * ndmpdp; i++) { + ((pd_entry_t *)DMPDphys)[i] = DMPGphys + (i << PAGE_SHIFT); + ((pd_entry_t *)DMPDphys)[i] |= + pmap_bits_default[PG_RW_IDX] | + pmap_bits_default[PG_V_IDX] | + /*pmap_bits_default[PG_PS_IDX] |*/ + pmap_bits_default[PG_G_IDX] | + pmap_bits_default[PG_M_IDX] | + pmap_bits_default[PG_A_IDX]; + } + + /* + * Populate 1GB PTEs (PDP pages), point to DMPD pages + */ + for (i = 0; i < ndmpdp; i++) { + ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (i << PAGE_SHIFT); + ((pdp_entry_t *)DMPDPphys)[i] |= + pmap_bits_default[PG_RW_IDX] | + pmap_bits_default[PG_V_IDX] | + pmap_bits_default[PG_A_IDX]; } -#endif /* And recursively map PML4 to itself in order to get PTmap */ ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys; @@ -1706,6 +1697,21 @@ pmap_page_init(struct vm_page *m) * Low level helper routines..... ***************************************************/ +/* + * Returns the DMAP address of the page. If the page's physical address + * is not in the DMAP, returns 0 + */ +vm_paddr_t +pmap_page_to_dmap(vm_page_t m) +{ + vm_paddr_t pa; + + pa = VM_PAGE_TO_PHYS(m); + if (pa < DMapMaxAddress - DMAP_MIN_ADDRESS) + return PHYS_TO_DMAP(pa); + return 0; +} + /* * Extract the physical page address associated with the map/VA pair. * The page must be wired for this to work reliably. @@ -2058,36 +2064,115 @@ pmap_map(vm_offset_t *virtp, vm_paddr_t start, vm_paddr_t end, int prot) return va_start; } +vm_offset_t +pmap_tempmap_enter(vm_paddr_t pa, int memattr) +{ + struct mdglobaldata *md = mdcpu; + pt_entry_t pte; + + if (md->gd_temp_pte == NULL) { + vm_offset_t va; + + va = kmem_alloc_nofault(kernel_map, PAGE_SIZE, + VM_SUBSYS_SYSMAP, PAGE_SIZE); + crit_enter(); + if (md->gd_temp_pte == NULL) { + md->gd_temp_pte = vtopte(va); + md->gd_temp_va = va; + } else { + kmem_free(kernel_map, va, PAGE_SIZE); + } + crit_exit(); + } + crit_enter(); + + pte = pa | + kernel_pmap->pmap_bits[PG_RW_IDX] | + kernel_pmap->pmap_bits[PG_V_IDX] | + kernel_pmap->pmap_cache_bits_pte[memattr]; + *md->gd_temp_pte = pte; + cpu_invlpg((void *)md->gd_temp_va); + return (md->gd_temp_va); +} + +void +pmap_tempmap_exit(void) +{ + struct mdglobaldata *md = mdcpu; + + *md->gd_temp_pte = 0; + cpu_invlpg((void *)md->gd_temp_va); + crit_exit(); +} + #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) /* - * Remove the specified set of pages from the data and instruction caches. + * Invalidate a single page, memory barrier is not provided + */ +void +pmap_invalidate_cache_page(vm_page_t m) +{ + vm_offset_t daddr, eva, pa; + + pa = VM_PAGE_TO_PHYS(m); + if (pa < DMapMaxAddress - DMAP_MIN_ADDRESS) { + daddr = PHYS_TO_DMAP(pa); + eva = daddr + PAGE_SIZE; + while (daddr < eva) { + clflush(daddr); + daddr += cpu_clflush_line_size; + } + } else { + daddr = pmap_tempmap_enter(pa, VM_MEMATTR_UNCACHEABLE); + eva = daddr + PAGE_SIZE; + while (daddr < eva) { + clflush(daddr); + daddr += cpu_clflush_line_size; + } + pmap_tempmap_exit(); + } +} + + +/* + * Flush and remove the specified set of pages from the data and instruction + * caches, and provide memory barriers. * - * In contrast to pmap_invalidate_cache_range(), this function does not - * rely on the CPU's self-snoop feature, because it is intended for use - * when moving pages into a different cache domain. + * If the range is too large we just issue wbinvd on all cores, else we + * iterate the range and issue clflush(). */ void pmap_invalidate_cache_pages(vm_page_t *pages, int count) { - vm_offset_t daddr, eva; - int i; - +#if 0 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || (cpu_feature & CPUID_CLFSH) == 0) - wbinvd(); - else { +#else + if ((cpu_feature & CPUID_CLFSH) == 0) +#endif + { + if (ncpus == 1) + wbinvd(); + else + cpu_wbinvd_on_all_cpus(); + //wbinvd(); + } else { + int i; + cpu_mfence(); for (i = 0; i < count; i++) { - daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i])); - eva = daddr + PAGE_SIZE; - for (; daddr < eva; daddr += cpu_clflush_line_size) - clflush(daddr); + pmap_invalidate_cache_page(pages[i]); } cpu_mfence(); } } +/* + * Invalidate cache VA range. If the range is too large we just + * issue wbinvd on all cores, else we iterate the range and issue + * clflush(). + */ void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) { @@ -2096,12 +2181,44 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) KASSERT((eva & PAGE_MASK) == 0, ("pmap_invalidate_cache_range: eva not page-aligned")); + +#if 1 +#if 0 + if (eva - sva >= PMAP_CLFLUSH_THRESHOLD || + (cpu_feature & CPUID_CLFSH) == 0) +#else + if ((cpu_feature & CPUID_CLFSH) == 0) +#endif + { + if (ncpus == 1) + wbinvd(); + else + cpu_wbinvd_on_all_cpus(); + //wbinvd(); + } else { + //kprintf("sva %016lx eva %016lx lsize %d\n", + // sva, eva, cpu_clflush_line_size); + //print_backtrace(5); + cpu_mfence(); + while (sva < eva) { + clflush(sva); + sva += cpu_clflush_line_size; + //kprintf("x"); + } + cpu_mfence(); + //kprintf("ok\n"); + } +#else if (cpu_feature & CPUID_SS) { ; /* If "Self Snoop" is supported, do nothing. */ } else { /* Globally invalidate caches */ - cpu_wbinvd_on_all_cpus(); + if (ncpus == 1) + wbinvd(); + else + cpu_wbinvd_on_all_cpus(); } +#endif } /* @@ -2126,7 +2243,8 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) * The page *must* be wired. */ static __inline void -_pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count, int doinval) +_pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count, int doinval, + int memattr) { vm_offset_t end_va; vm_offset_t va; @@ -2136,30 +2254,64 @@ _pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count, int doinval) for (va = beg_va; va < end_va; va += PAGE_SIZE) { pt_entry_t pte; pt_entry_t *ptep; + int pat_mode = (memattr >= 0) ? memattr : (*m)->pat_mode; ptep = vtopte(va); pte = VM_PAGE_TO_PHYS(*m) | kernel_pmap->pmap_bits[PG_RW_IDX] | kernel_pmap->pmap_bits[PG_V_IDX] | - kernel_pmap->pmap_cache_bits_pte[(*m)->pat_mode]; + kernel_pmap->pmap_cache_bits_pte[pat_mode]; // pgeflag; atomic_swap_long(ptep, pte); + if (doinval == 2) + cpu_invlpg((void *)va); m++; } - if (doinval) + if (doinval == 1) pmap_invalidate_range(kernel_pmap, beg_va, end_va); } +/* + * Enter pte for current cpu use only + */ +void +pmap_qenter_quick(vm_offset_t beg_va, vm_page_t *m, int count) +{ + _pmap_qenter(beg_va, m, count, 2, -1); +} + +/* + * Enter pte for any cpu use + */ void pmap_qenter(vm_offset_t beg_va, vm_page_t *m, int count) { - _pmap_qenter(beg_va, m, count, 1); + _pmap_qenter(beg_va, m, count, 1, -1); +} + +/* + * Enter pte for any cpu use, override pat_mode with memattr for entry + */ +void +pmap_qenter_memattr(vm_offset_t beg_va, vm_page_t *m, int count, int memattr) +{ + _pmap_qenter(beg_va, m, count, 1, memattr); +} + +/* + * Enter pte for current cpu only, override pat_mode with memattr for entry + */ +void +pmap_qenter_quick_memattr(vm_offset_t beg_va, vm_page_t *m, + int count, int memattr) +{ + _pmap_qenter(beg_va, m, count, 2, memattr); } void pmap_qenter_noinval(vm_offset_t beg_va, vm_page_t *m, int count) { - _pmap_qenter(beg_va, m, count, 0); + _pmap_qenter(beg_va, m, count, 0, -1); } /* @@ -6086,7 +6238,8 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) pa += PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, va, va + size); - pmap_invalidate_cache_range(va, va + size); + //pmap_invalidate_cache_range(va, va + size); + cpu_wbinvd_on_all_cpus(); return ((void *)(va + offset)); } @@ -6104,79 +6257,134 @@ pmap_unmapdev(vm_offset_t va, vm_size_t size) } /* - * Sets the memory attribute for the specified page. + * Sets the memory attribute for the specified page. If the attribute + * is not normal caching, we also flush the related cache lines. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { + vm_offset_t pa; + + m->pat_mode = ma; - m->pat_mode = ma; + /* + * If "m" is a normal page, update its pte in the DMAP. This update + * can be relied upon to perform any cache operations that are + * required for data coherence. + * + * fictitious pages can still be associated with system memory + * via the low-memory dma reserve. + */ + pa = VM_PAGE_TO_PHYS(m); + if (pa < DMapMaxAddress - DMAP_MIN_ADDRESS) + pmap_change_attr(PHYS_TO_DMAP(pa), 1, m->pat_mode); - /* - * If "m" is a normal page, update its direct mapping. This update - * can be relied upon to perform any cache operations that are - * required for data coherence. - */ - if ((m->flags & PG_FICTITIOUS) == 0) - pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 1, m->pat_mode); + /* + * Invalidate the cpu cache, even when changing back to a cacheable + * state, because apparently it is possible for DMA to modify the + * memory after a speculative read (even on uncached memory) loaded + * the cache, resultin in a stale cache. + */ + pmap_invalidate_cache_pages(&m, 1); } /* * Change the PAT attribute on an existing kernel memory map. Caller * must ensure that the virtual memory in question is not accessed * during the adjustment. - * - * If the va is within the DMAP we cannot use vtopte() because the DMAP - * utilizes 2MB or 1GB pages. 2MB is forced atm so calculate the pd_entry - * pointer based on that. */ void pmap_change_attr(vm_offset_t va, vm_size_t count, int mode) { pt_entry_t *pte; vm_offset_t base; - int changed = 0; + vm_offset_t pa; + vm_offset_t dmva_beg; + vm_offset_t dmva_end; if (va == 0) panic("pmap_change_attr: va is NULL"); base = trunc_page(va); - if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { - pd_entry_t *pd; + dmva_beg = 0; + dmva_end = 0; - KKASSERT(va < DMapMaxAddress); - pd = (pd_entry_t *)PHYS_TO_DMAP(DMPDphys); - pd += (va - DMAP_MIN_ADDRESS) >> PDRSHIFT; + if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { + KKASSERT(va + count * PAGE_SIZE < DMapMaxAddress); + pte = (pt_entry_t *)PHYS_TO_DMAP(DMPGphys); + pte += (va - DMAP_MIN_ADDRESS) >> PAGE_SHIFT; while ((long)count > 0) { - *pd = - (*pd & ~(pd_entry_t)(kernel_pmap->pmap_cache_mask_pde)) | - kernel_pmap->pmap_cache_bits_pde[mode]; - count -= NBPDR / PAGE_SIZE; - va += NBPDR; - ++pd; + *pte = + (*pte & ~(pd_entry_t)(kernel_pmap->pmap_cache_mask_pte)) | + kernel_pmap->pmap_cache_bits_pte[mode]; + --count; + va += PAGE_SIZE; + ++pte; } } else { while (count) { + /* + * Adjust the kernel map PTE + */ pte = vtopte(va); *pte = (*pte & ~(pt_entry_t)(kernel_pmap->pmap_cache_mask_pte)) | kernel_pmap->pmap_cache_bits_pte[mode]; + + /* + * If the physical address is in physical memory, + * also adjust the DMAP PTE. + */ + pa = *pte & PG_FRAME; + if (pa < DMapMaxAddress - DMAP_MIN_ADDRESS) { + pte = (pt_entry_t *)PHYS_TO_DMAP(DMPGphys); + pte += pa >> PAGE_SHIFT; + *pte = (*pte & ~(pd_entry_t)( + kernel_pmap->pmap_cache_mask_pte)) | + kernel_pmap->pmap_cache_bits_pte[mode]; + + /* + * Collect invalidation info for the DMAP + */ + if (dmva_beg == dmva_end) { + dmva_beg = PHYS_TO_DMAP(pa); + dmva_end = dmva_beg + PAGE_SIZE; + } else if (dmva_end == PHYS_TO_DMAP(pa)) { + dmva_end += PAGE_SIZE; + } else { + pmap_inval_smp(kernel_pmap, dmva_beg, + (dmva_end - dmva_beg) >> + PAGE_SHIFT, + NULL, 0); + dmva_beg = PHYS_TO_DMAP(pa); + dmva_end = dmva_beg + PAGE_SIZE; + } + } --count; va += PAGE_SIZE; } } - changed = 1; /* XXX: not optimal */ - /* - * Flush CPU caches if required to make sure any data isn't cached that - * shouldn't be, etc. + * SMP synchronize the page tables. And if changing to non-cached + * mode we also flush the CPU caches for the range. */ - if (changed) { - pmap_invalidate_range(kernel_pmap, base, va); - pmap_invalidate_cache_range(base, va); + if (dmva_beg != dmva_end) { + pmap_inval_smp(kernel_pmap, dmva_beg, + (dmva_end - dmva_beg) >> PAGE_SHIFT, + NULL, 0); } + + pmap_invalidate_range(kernel_pmap, base, va); + + /* + * Invalidate the cpu cache, even when changing back to a cacheable + * state, because apparently it is possible for DMA to modify the + * memory after a speculative read (even on uncached memory) loaded + * the cache, resultin in a stale cache. + */ + pmap_invalidate_cache_range(base, va); } /* diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 6e007ebb37..18f104a72a 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -167,8 +167,13 @@ struct lock { * * The first three flags may be set in lock_init to set their mode permanently, * or passed in as arguments to the lock manager. + * + * NOTE: LK_SPIN does not disable interrupts so if the lock is competing with + * an interrupt the caller should not use the lockmgr LK_SPIN mode. + * + * NOTE: LK_SPIN ignores timouts */ -#define LK_EXTFLG_MASK 0x070000F0 /* mask of external flags */ +#define LK_EXTFLG_MASK 0x0F0000F0 /* mask of external flags */ #define LK_NOWAIT 0x00000010 /* do not sleep to await lock */ #define LK_SLEEPFAIL 0x00000020 /* sleep, then return failure */ #define LK_CANRECURSE 0x00000040 /* allow recursive exclusive lock */ @@ -176,6 +181,7 @@ struct lock { #define LK_CANCELABLE 0x01000000 /* blocked caller can be canceled */ #define LK_TIMELOCK 0x02000000 #define LK_PCATCH 0x04000000 /* timelocked with signal catching */ +#define LK_SPIN 0x08000000 /* spin instead of sleep */ /* * Control flags diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index ea9c2ab3c5..46788878ad 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -184,6 +184,7 @@ void pmap_enter (pmap_t, vm_offset_t, struct vm_page *, void pmap_maybethreaded(pmap_t); int pmap_mapped_sync(vm_page_t m); vm_page_t pmap_fault_page_quick(pmap_t, vm_offset_t, vm_prot_t, int *); +vm_paddr_t pmap_page_to_dmap(vm_page_t m); vm_paddr_t pmap_extract (pmap_t pmap, vm_offset_t va, void **handlep); void pmap_extract_done (void *handle); void pmap_growkernel (vm_offset_t, vm_offset_t); @@ -206,6 +207,9 @@ void pmap_npt_transform (pmap_t, int); void pmap_protect (pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); void pmap_remove_specific (pmap_t, vm_page_t); void pmap_qenter (vm_offset_t, struct vm_page **, int); +void pmap_qenter_memattr (vm_offset_t, struct vm_page **, int, int); +void pmap_qenter_quick (vm_offset_t, struct vm_page **, int); +void pmap_qenter_quick_memattr (vm_offset_t, struct vm_page **, int, int); void pmap_qenter_noinval (vm_offset_t, struct vm_page **, int); void pmap_qremove (vm_offset_t, int); void pmap_qremove_quick (vm_offset_t, int); diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index e43203f486..b5ed1b1260 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -139,7 +139,7 @@ struct vpgqueues vm_page_queues[PQ_COUNT]; static volatile int vm_pages_waiting; static struct alist vm_contig_alist; -static struct almeta vm_contig_ameta[ALIST_RECORDS_65536]; +static struct almeta vm_contig_ameta[ALIST_RECORDS_1048576]; static struct spinlock vm_contig_spin = SPINLOCK_INITIALIZER(&vm_contig_spin, "vm_contig_spin"); __read_mostly static int vm_page_hash_vnode_only; @@ -248,7 +248,7 @@ vm_add_new_page(vm_paddr_t pa, int *badcountp) * which we consider bugs... but don't crash). Note that m->phys_addr * is pre-initialized, so use m->queue as a check. */ - if (m->queue) { + if (m->flags & PG_ADDED) { if (*badcountp < 10) { kprintf("vm_add_new_page: duplicate pa %016jx\n", (intmax_t)pa); @@ -261,7 +261,7 @@ vm_add_new_page(vm_paddr_t pa, int *badcountp) } m->phys_addr = pa; - m->flags = 0; + m->flags = PG_ADDED; m->pat_mode = PAT_WRITE_BACK; m->pc = (pa >> PAGE_SHIFT); @@ -292,20 +292,19 @@ vm_add_new_page(vm_paddr_t pa, int *badcountp) m->wire_count = 1; atomic_add_long(&vmstats.v_wire_count, 1); alist_free(&vm_contig_alist, pa >> PAGE_SHIFT, 1); - return; - } + } else { + /* + * General page + */ + m->queue = m->pc + PQ_FREE; + KKASSERT(m->dirty == 0); - /* - * General page - */ - m->queue = m->pc + PQ_FREE; - KKASSERT(m->dirty == 0); - - atomic_add_long(&vmstats.v_page_count, 1); - atomic_add_long(&vmstats.v_free_count, 1); - vpq = &vm_page_queues[m->queue]; - TAILQ_INSERT_HEAD(&vpq->pl, m, pageq); - ++vpq->lcnt; + atomic_add_long(&vmstats.v_page_count, 1); + atomic_add_long(&vmstats.v_free_count, 1); + vpq = &vm_page_queues[m->queue]; + TAILQ_INSERT_HEAD(&vpq->pl, m, pageq); + ++vpq->lcnt; + } } /* @@ -423,7 +422,7 @@ vm_page_startup(void) * * By default, 128M is left in reserve on machines with 2G+ of ram. */ - vm_low_phys_reserved = (vm_paddr_t)65536 << PAGE_SHIFT; + vm_low_phys_reserved = (vm_paddr_t)524288 << PAGE_SHIFT; if (vm_low_phys_reserved > total / 4) vm_low_phys_reserved = total / 4; if (vm_dma_reserved == 0) { @@ -432,8 +431,8 @@ vm_page_startup(void) vm_dma_reserved = total / 16; } #endif - alist_init(&vm_contig_alist, 65536, vm_contig_ameta, - ALIST_RECORDS_65536); + alist_init(&vm_contig_alist, 1048576, vm_contig_ameta, + ALIST_RECORDS_1048576); /* * Initialize the mem entry structures now, and put them in the free @@ -2860,6 +2859,7 @@ vm_page_free_contig(vm_page_t m, unsigned long size) vm_paddr_t pa = VM_PAGE_TO_PHYS(m); vm_pindex_t start = pa >> PAGE_SHIFT; vm_pindex_t pages = (size + PAGE_MASK) >> PAGE_SHIFT; + vm_pindex_t i; if (vm_contig_verbose) { kprintf("vm_page_free_contig: %016jx/%ldk\n", @@ -2872,6 +2872,20 @@ vm_page_free_contig(vm_page_t m, unsigned long size) KKASSERT(m->wire_count == 1); KKASSERT(m->flags & PG_FICTITIOUS); KKASSERT(pa + size <= vm_low_phys_reserved); + for (i = 0; i < pages; ++i) { + /* + * Reset state to invalidate, not dirty, normal + * cpu caching + */ + vm_page_t p = &m[i]; + + p->valid = 0; + vm_page_undirty(p); + if (p->pat_mode != PAT_WRITE_BACK) { + p->pat_mode = PAT_WRITE_BACK; + pmap_page_set_memattr(p, PAT_WRITE_BACK); + } + } spin_lock(&vm_contig_spin); alist_free(&vm_contig_alist, start, pages); spin_unlock(&vm_contig_spin); @@ -3189,8 +3203,15 @@ vm_page_free_toq(vm_page_t m) vm_page_and_queue_spin_lock(m); _vm_page_rem_queue_spinlocked(m); + /* + * Reset state to invalidate, not dirty, normal cpu caching + */ m->valid = 0; vm_page_undirty(m); + if (m->pat_mode != PAT_WRITE_BACK) { + m->pat_mode = PAT_WRITE_BACK; + pmap_page_set_memattr(m, PAT_WRITE_BACK); + } if (m->wire_count != 0) { if (m->wire_count > 1) { diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 9bf13ce03c..874117abc4 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -197,8 +197,9 @@ struct vm_page { uint8_t valid; /* map of valid DEV_BSIZE chunks */ uint8_t dirty; /* map of dirty DEV_BSIZE chunks */ uint32_t flags; /* see below */ - int unused01; /* available */ + int ext_refs; /* extension - kptr caching refs */ /* 128 bytes */ + void *ext_kptr; /* extension - cached kptr */ #ifdef VM_PAGE_DEBUG const char *busy_func; int busy_line; @@ -387,6 +388,7 @@ extern struct vpgqueues vm_page_queues[PQ_COUNT]; #define PG_UNUSED10000 0x00010000 #define PG_UNUSED20000 0x00020000 #define PG_NEED_COMMIT 0x00040000 /* clean page requires commit */ +#define PG_ADDED 0x00080000 /* added via vm_add_new_pages */ #define PG_KEEP_NEWPAGE_MASK (0)