/**************************************************************************** * * * Copyright 1999-2005 ATI Technologies Inc., Markham, Ontario, CANADA. * * All Rights Reserved. * * * * Your use and or redistribution of this software in source and \ or * * binary form, with or without modification, is subject to: (i) your * * ongoing acceptance of and compliance with the terms and conditions of * * the ATI Technologies Inc. software End User License Agreement; and (ii) * * your inclusion of this notice in any version of this software that you * * use or redistribute. A copy of the ATI Technologies Inc. software End * * User License Agreement is included with this software and is also * * available by contacting ATI Technologies Inc. at http://www.ati.com * * * ****************************************************************************/ #ifdef __KERNEL__ #ifndef MODULE !!! This is not currently supported, !!! since it requires changes to linux/init/main.c. #endif /* !MODULE */ // ============================================================ #include #ifdef MODVERSIONS #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,71) #include #endif #endif #include #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,71) #define EXPORT_SYMTAB 1 #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,71) #if !defined(CONFIG_X86_PC) #if !defined(CONFIG_X86_64) #if !defined(CONFIG_X86_VOYAGER) #if !defined(CONFIG_X86_NUMAQ) #if !defined(CONFIG_X86_SUMMIT) #if !defined(CONFIG_X86_BIGSMP) #if !defined(CONFIG_X86_VISWS) #if !defined(CONFIG_X86_GENERICARCH) #error unknown or undefined architecture configured #endif #endif #endif #endif #endif #endif #endif #endif #endif /* LINUX_VERSION_CODE */ /* The dirty-page-tracking patch included in NLD 9 SMP kernels defines * a static inline function that uses a GPL-only symbol in a header * file. Therefore any non-GPL module built against such a kernel * configuration is broken and cannot be loaded. We work around that * problem by disabling the respective kernel configuration option for * our module build. * * This will break page tracking when this kernel module is * used. However, on a standard system page tracking is disabled * anyways. It is only activated and used by specific in-kernel agents * for example for CPU hot-plugging. I wonder why a desktop * distribution would even include such a kernel patch. */ #ifdef CONFIG_MEM_MIRROR /* Prevent asm/mm_track.h from being included in subsequent * kernel headers as that would redefine CONFIG_MEM_MIRROR. */ #ifndef CONFIG_X86_64 #define __I386_MMTRACK_H__ #define mm_track(ptep) #else #define __X86_64_MMTRACK_H__ #define mm_track_pte(ptep) #define mm_track_pmd(ptep) #define mm_track_pgd(ptep) #define mm_track_pml4(ptep) #define mm_track_phys(x) #endif #warning "Disabling CONFIG_MEM_MIRROR because it does not work with non-GPL modules." #warning "This will break page tracking when the fglrx kernel module is used." #undef CONFIG_MEM_MIRROR #endif /* CONFIG_MEM_MIRROR */ // ============================================================ // always defined #define __AGP__BUILTIN__ //#define FGL_USE_SCT /* for developer use only */ // ============================================================ #include /* for installing the patch wrapper */ #include #include #include #include #include #include #include #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,71) #include #endif #include #include #include // newer SuSE kernels need this #include #include // for lock_page and unlock_page #include #if defined(__ia64__) #include #endif #include #include #include //#include #include #include #include #include #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) #include // for flush_tlb_page #else #include // for flush_tlb_page #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,71) #include #endif #ifdef CONFIG_MTRR #include #endif #include #include #ifndef EXPORT_NO_SYMBOLS #define EXPORT_NO_SYMBOLS #endif #ifdef __x86_64__ #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) #include "linux/ioctl32.h" #else #include "asm/ioctl32.h" #endif #ifdef __x86_64__ #include "asm/compat.h" #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,2) #include "linux/syscalls.h" #endif #endif // For 2.6.18 or higher, the UTS_RELEASE is defined in the linux/utsrelease.h. #ifndef UTS_RELEASE #include #endif #if defined(__i386__) #ifndef do_div #include "asm/div64.h" #endif #endif #include #include "firegl_public.h" // ============================================================ #ifndef TRUE #define TRUE 1 #endif #ifndef FALSE #define FALSE 0 #endif #ifndef NULL #define NULL (void*)0 #endif // VM_SHM is deleted in 2.6.18 or higher kernels. #ifndef VM_SHM #define VM_SHM 0 #endif #ifdef FGL_USE_SCT // get direct function pointers from sys_call_table for calling #else /* FGL_USE_SCT */ // call functions indirectly by using the syscall macros, // entrypoints get defined by below constructs #if !defined(__ia64__) // the macros do use errno variable int errno; #endif // __ia64__ #if defined(__i386__) #define __syscall_return(type, res) \ do { \ if ((unsigned long)(res) >= (unsigned long)(-(128 + 1))) { \ errno = -(res); \ res = -1; \ } \ return (type) (res); \ } while (0) #define _syscall2(type,name,type1,arg1,type2,arg2) \ type name(type1 arg1,type2 arg2) \ { \ long __res; \ __asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \ : "=a" (__res) \ : "0" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)) \ : "memory"); \ __syscall_return(type,__res); \ } #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ type name(type1 arg1,type2 arg2,type3 arg3) \ { \ long __res; \ __asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \ : "=a" (__res) \ : "0" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)), \ "d" ((long)(arg3)) : "memory"); \ __syscall_return(type,__res); \ } #elif defined(__x86_64__) #define __syscall_clobber "r11","rcx","memory" #define __syscall "syscall" #define __syscall_return(type, res) \ do { \ if ((unsigned long)(res) >= (unsigned long)(-127)) { \ errno = -(res); \ res = -1; \ } \ return (type) (res); \ } while (0) #define _syscall2(type,name,type1,arg1,type2,arg2) \ type name(type1 arg1,type2 arg2) \ { \ long __res; \ __asm__ volatile (__syscall \ : "=a" (__res) \ : "0" (__NR_##name),"D" ((long)(arg1)),"S" ((long)(arg2)) : __syscall_clobber ); \ __syscall_return(type,__res); \ } #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ type name(type1 arg1,type2 arg2,type3 arg3) \ { \ long __res; \ __asm__ volatile (__syscall \ : "=a" (__res) \ : "0" (__NR_##name),"D" ((long)(arg1)),"S" ((long)(arg2)), \ "d" ((long)(arg3)) : __syscall_clobber); \ __syscall_return(type,__res); \ } #endif // int mlock(const void *addr, size_t len); _syscall2(int, mlock, const void *, addr, size_t, len ) // int munlock(const void *addr, size_t len); _syscall2(int, munlock, const void *, addr, size_t, len ) #if !defined(__ia64__) #if !defined(__x86_64__) // TODO: ia64 // int modify_ldt(int func, void *ptr, unsigned long bytecount); _syscall3( int, modify_ldt, int, func, void *, ptr, unsigned long, bytecount ) #endif #endif #endif /* FGL_USE_SCT */ #ifdef FGL_LINUX253P1_VMA_API // Linux 2.5.3-pre1 and compatibles #define FGL_VMA_API_TYPE struct vm_area_struct * #define FGL_VMA_API_NAME vma #define FGL_VMA_API_PROTO FGL_VMA_API_TYPE FGL_VMA_API_NAME, #define FGL_VMA_API_PASS FGL_VMA_API_NAME, #else /* FGL_253P1_VMA_API */ // Linux 2.4.0 and compatibles #define FGL_VMA_API_TYPE /* none */ #define FGL_VMA_API_NAME /* none */ #define FGL_VMA_API_PROTO /* none */ #define FGL_VMA_API_PASS /* none */ #endif /* FGL_253P1_VMA_API */ #ifndef preempt_disable #define preempt_disable() #define preempt_enable() #endif // ============================================================ /* globals */ char* firegl = NULL; int __ke_debuglevel = 0; int __ke_moduleflags = 0; /* global module vars and constants - defined trough macros */ MODULE_AUTHOR("Fire GL - ATI Research GmbH, Germany"); MODULE_DESCRIPTION("ATI Fire GL"); #ifdef MODULE_PARM MODULE_PARM(firegl, "s"); #else module_param(firegl, charp, 0); #endif #ifdef MODULE_LICENSE MODULE_LICENSE("Proprietary. (C) 2002 - ATI Technologies, Starnberg, GERMANY"); #endif #define FIREGL_KERNEL_DRIVER_NAME "fglrx" // needed for kernel message macros /* globals constants */ const char* __ke_UTS_RELEASE = UTS_RELEASE; const unsigned int __ke_PAGE_SHIFT = PAGE_SHIFT; const unsigned int __ke_PAGE_SIZE = PAGE_SIZE; const unsigned long __ke_PAGE_MASK = PAGE_MASK; const unsigned long __ke_LINUX_VERSION_CODE = LINUX_VERSION_CODE; // create global constants and hint symbols (i.e. for objdump checking) #ifdef MODVERSIONS const unsigned long __ke_MODVERSIONS_State = 1; const char BUILD_KERNEL_HAS_MODVERSIONS_SET; #else const unsigned long __ke_MODVERSIONS_State = 0; const char BUILD_KERNEL_HAS_MODVERSIONS_CLEARED; #endif #ifdef __SMP__ const unsigned long __ke_SMP_State = 1; const char BUILD_KERNEL_HAS_SMP_SET; #else const unsigned long __ke_SMP_State = 0; const char BUILD_KERNEL_HAS_SMP_CLEARED; #endif /* PAE is always disabled if it's not x86_64 or CONFIG_X86_PAE is disabled on a 32 bit system.*/ #if !defined(__x86_64__) && !defined(CONFIG_X86_PAE) const unsigned long __ke_PAE_State = 0; #else const unsigned long __ke_PAE_State = 1; #endif /* globals vars that are in fact constants */ unsigned long __ke_HZ; // ============================================================ /* global structures */ int ip_firegl_open(struct inode* inode, struct file* filp) { return firegl_open(inode, filp); } int ip_firegl_release(struct inode* inode, struct file* filp) { return firegl_release(inode, filp); } int ip_firegl_ioctl(struct inode* inode, struct file* filp, unsigned int cmd, unsigned long arg) { return firegl_ioctl(inode, filp, cmd, arg); } int ip_firegl_mmap(struct file* filp, struct vm_area_struct* vma) { return firegl_mmap(filp, vma); } #if defined(FIREGL_IOCTL_COMPAT) && defined(__x86_64__) long ip_firegl_compat_ioctl(struct file* filp, unsigned int cmd, unsigned long arg) { return firegl_compat_ioctl(filp, cmd, arg); } #endif static struct file_operations firegl_fops = { #ifdef THIS_MODULE owner: THIS_MODULE, #endif open: ip_firegl_open, release: ip_firegl_release, ioctl: ip_firegl_ioctl, mmap: ip_firegl_mmap, #if defined(FIREGL_IOCTL_COMPAT) && defined(__x86_64__) compat_ioctl: ip_firegl_compat_ioctl, #endif }; typedef struct { __ke_device_t pubdev; // MUST BE FIRST MEMBER dev_t device; // Device number for mknod /* Locking */ spinlock_t spinlock[__KE_MAX_SPINLOCKS]; /* For inuse, open_count, buf_use */ struct semaphore struct_sem[__KE_MAX_SEMAPHORES]; /* For linked list manipulations */ sigset_t sigmask; } device_t; static device_t firegl_public_device; // The Fire GL public device /*****************************************************************************/ // standard XFree86 DRM proc support #define DRM(x) FGLDRM_##x #include "drm.h" // mem_info() is missing in drm_proc.h. But, it is a DRM design problem anyway! // The first registered DRM device could never report memory statisticts of another // DRM device, cause the DRM mem_info routine uses local variables. So, let's use a dummy. static int DRM(mem_info)(char *buf __attribute__((unused)), char **start __attribute__((unused)), off_t offset __attribute__((unused)), int len __attribute__((unused)), int *eof, void *data __attribute__((unused))) { *eof = 1; return 0; } #if 0 #ifdef vmalloc_to_page #undef vmalloc_to_page #endif #define vmalloc_to_page drm_inline_vmalloc_to_page #endif #include "drm_proc.h" #ifndef DRM_MAJOR #define DRM_MAJOR 226 #endif // DRM_MAJOR static __ke_proc_list_t *drm_proclist = NULL; /*****************************************************************************/ // Fire GL DRM stub support (compatible with standard DRM stub) #define FIREGL_STUB_MAXCARDS 16 typedef struct firegl_stub_list_tag { const char *name; struct file_operations *fops; struct proc_dir_entry *dev_root; __ke_proc_list_t *proclist; } firegl_stub_list_t; static firegl_stub_list_t firegl_stub_list[FIREGL_STUB_MAXCARDS]; static struct proc_dir_entry *firegl_stub_root; static int firegl_minor; typedef struct firegl_drm_stub_info_tag { int (*info_register)(const char *name, struct file_operations *fops, device_t *dev); int (*info_unregister)(int minor); unsigned long signature; // to check for compatible Fire GL DRM device } firegl_drm_stub_info_t; static firegl_drm_stub_info_t firegl_stub_info; static char *__ke_pte_phys_addr_str(pte_t pte, char *buf, __ke_dma_addr_t* phys_address); unsigned long ATI_API_CALL __ke_cpu_to_le32(unsigned long _u) { return cpu_to_le32(_u); } unsigned long long ATI_API_CALL __ke_cpu_to_le64(unsigned long long _u) { return cpu_to_le64(_u); } #define READ_PROC_WRAP(func) \ static int func##_wrap(char *buf, char **start, __ke_off_t offset, \ int len, int* eof, void* data) \ { \ return func(buf, start, offset, len, eof, data); \ } READ_PROC_WRAP(drm_name_info) READ_PROC_WRAP(drm_mem_info) READ_PROC_WRAP(drm_mem_info1) READ_PROC_WRAP(drm_vm_info) READ_PROC_WRAP(drm_clients_info) READ_PROC_WRAP(firegl_lock_info) READ_PROC_WRAP(firegl_umm_info) #ifdef DEBUG READ_PROC_WRAP(drm_bq_info) READ_PROC_WRAP(firegl_debug_info) #endif READ_PROC_WRAP(firegl_bios_version) static int firegl_interrupt_open_wrap( struct inode *inode, struct file *file) { return firegl_interrupt_open(inode, file); } static int firegl_interrupt_release_wrap( struct inode *inode, struct file *file) { return firegl_interrupt_release(inode, file); } static ssize_t firegl_interrupt_read_wrap( struct file *user_file, char __user *user_buf, size_t user_buf_size, loff_t *user_file_pos) { return (ssize_t) firegl_interrupt_read(user_file, user_buf, user_buf_size, user_file_pos); } static unsigned int firegl_interrupt_poll_wrap(struct file *user_file, poll_table *pt) { if(firegl_interrupt_poll(user_file, (__ke_poll_table*)pt)) { return POLLIN | POLLRDNORM; } else { return 0; } } static ssize_t firegl_interrupt_write_wrap( struct file *user_file, char __user *user_buf, size_t user_buf_size, loff_t *user_file_pos) { return (ssize_t) firegl_interrupt_write(user_file, user_buf, user_buf_size, user_file_pos); } static struct file_operations firegl_interrupt_file_ops = { .open = firegl_interrupt_open_wrap, .read = firegl_interrupt_read_wrap, .release = firegl_interrupt_release_wrap, .poll = firegl_interrupt_poll_wrap, .write = firegl_interrupt_write_wrap }; __ke_proc_list_t firegl_proc_list[] = { { "name", drm_name_info_wrap, NULL}, { "mem", drm_mem_info_wrap, NULL}, { "mem1", drm_mem_info1_wrap, NULL}, { "vm", drm_vm_info_wrap, NULL}, { "clients", drm_clients_info_wrap, NULL}, { "lock", firegl_lock_info_wrap, NULL}, { "umm", firegl_umm_info_wrap, NULL}, #ifdef DEBUG { "bq_info", drm_bq_info_wrap, NULL}, { "debug", firegl_debug_info_wrap, NULL}, #endif { "biosversion", firegl_bios_version_wrap, NULL}, { "interrupts", NULL, (__ke_file_operations_t*)&firegl_interrupt_file_ops}, { "NULL", NULL, NULL} // Terminate List!!! }; static struct proc_dir_entry *firegl_proc_init( device_t *dev, int minor, struct proc_dir_entry *root, struct proc_dir_entry **dev_root, __ke_proc_list_t *proc_list ) // proc_list must be terminated! { struct proc_dir_entry *ent; char name[64]; __ke_proc_list_t *list = proc_list; __KE_DEBUG("minor %d, proc_list 0x%08lx\n", minor, (unsigned long)proc_list); if (!minor) root = create_proc_entry("dri", S_IFDIR, NULL); if (!root) { __KE_ERROR("Cannot create /proc/dri\n"); return NULL; } sprintf(name, "%d", minor); *dev_root = create_proc_entry(name, S_IFDIR, root); if (!*dev_root) { __KE_ERROR("Cannot create /proc/%s\n", name); return NULL; } while (list->f || list->fops) { ent = create_proc_entry(list->name, S_IFREG|S_IRUGO, *dev_root); if (!ent) { __KE_ERROR("Cannot create /proc/dri/%s/%s\n", name, list->name); while (proc_list != list) { remove_proc_entry(proc_list->name, *dev_root); proc_list++; } remove_proc_entry(name, root); if (!minor) remove_proc_entry("dri", NULL); return NULL; } if (list->f) { ent->read_proc = (read_proc_t*)list->f; } if (list->fops) { ent->proc_fops = (struct file_operations*)list->fops; } ent->data = (dev->pubdev.signature == FGL_DEVICE_SIGNATURE) ? ((void*)dev->pubdev.privdev) : (dev); list++; } return root; } static int firegl_proc_cleanup( int minor, struct proc_dir_entry *root, struct proc_dir_entry *dev_root, __ke_proc_list_t *proc_list ) { char name[64]; __KE_DEBUG("minor %d\n", minor); if (!root || !dev_root) { __KE_ERROR("no root\n"); return 0; } while (proc_list->f || proc_list->fops) { remove_proc_entry(proc_list->name, dev_root); proc_list++; } sprintf(name, "%d", minor); remove_proc_entry(name, root); if (!minor) remove_proc_entry("dri", NULL); return 0; } static int firegl_stub_open(struct inode *inode, struct file *filp) { #ifndef MINOR int minor = minor(inode->i_rdev); #else int minor = MINOR(inode->i_rdev); #endif int err = -ENODEV; struct file_operations *old_fops; __KE_DEBUG("\n"); if (minor >= FIREGL_STUB_MAXCARDS) return -ENODEV; if (!firegl_stub_list[minor].fops) return -ENODEV; old_fops = filp->f_op; filp->f_op = fops_get(firegl_stub_list[minor].fops); if (filp->f_op->open && (err = filp->f_op->open(inode, filp))) { fops_put(filp->f_op); filp->f_op = fops_get(old_fops); } fops_put(old_fops); return err; } static struct file_operations firegl_stub_fops = { owner: THIS_MODULE, open: firegl_stub_open }; static int firegl_stub_getminor(const char *name, struct file_operations *fops, device_t *dev) { int i; __KE_DEBUG("name=\"%s\"\n", name); for( i = 0; i < FIREGL_STUB_MAXCARDS; i++ ) { if( !firegl_stub_list[i].fops ) { firegl_stub_list[i].name = name; firegl_stub_list[i].fops = fops; firegl_stub_list[i].proclist = (dev->pubdev.signature == FGL_DEVICE_SIGNATURE) ? dev->pubdev.proclist : drm_proclist; firegl_stub_root = firegl_proc_init(dev, i, firegl_stub_root, &firegl_stub_list[i].dev_root, firegl_stub_list[i].proclist); __KE_DEBUG("minor=%d\n", i); return i; } } __KE_DEBUG("no more free minor\n"); return -1; } static int firegl_stub_putminor(int minor) { __KE_DEBUG("minor=%d\n", minor); if (minor < 0 || minor >= FIREGL_STUB_MAXCARDS) return -1; firegl_proc_cleanup(minor, firegl_stub_root, firegl_stub_list[minor].dev_root, firegl_stub_list[minor].proclist); firegl_stub_list[minor].name = NULL; firegl_stub_list[minor].fops = NULL; firegl_stub_list[minor].proclist = NULL; if( !minor ) { unregister_chrdev(DRM_MAJOR, "drm"); } return 0; } static int __init firegl_stub_register(const char *name, struct file_operations *fops, device_t *dev) { int err; __KE_DEBUG("name=\"%s\"\n", name); // try to register a drm char device for the firegl module err = register_chrdev(DRM_MAJOR, "drm", &firegl_stub_fops); if(err == 0) { __KE_DEBUG("register_chrdev() succeeded\n"); // register our own module handler will handle the DRM device firegl_stub_info.info_register = firegl_stub_getminor; firegl_stub_info.info_unregister = firegl_stub_putminor; } else if(err == -EINVAL) { __KE_ERROR("register_chrdev() failed with -EINVAL\n"); return -1; } else if(err == -EBUSY) { // the registering of the module's device has failed // because there was already some other drm module loaded. __KE_DEBUG("register_chrdev() failed with -EBUSY\n"); return -1; } else { __KE_ERROR("register_chrdev() failed with %i\n", err); return -1; } return firegl_stub_info.info_register(name, fops, dev); } static int __exit firegl_stub_unregister(int minor) { __KE_DEBUG("%d\n", minor); if (firegl_stub_info.info_unregister) return firegl_stub_info.info_unregister(minor); return -1; } #ifdef FIREGL_POWER_MANAGEMENT static struct pci_device_id fglrx_pci_table[] = { { .vendor = PCI_VENDOR_ID_ATI, .device = PCI_ANY_ID, .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, .class = (PCI_CLASS_DISPLAY_VGA << 8), .class_mask = ~0, }, { 0, } }; static int fglrx_pci_probe(struct pci_dev *dev, const struct pci_device_id *id_table) { return 0; } /* Starting from 2.6.14, kernel has new struct defined for pm_message_t, we have to handle this case separately. 2.6.11/12/13 kernels have pm_message_t defined as int and older kernels don't have pm_message_t defined. */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,14) static int fglrx_pci_suspend(struct pci_dev *pdev, pm_message_t pm_event) #else static int fglrx_pci_suspend(struct pci_dev *pdev, u32 pm_event) #endif { device_t* dev = &firegl_public_device; struct drm_device* privdev; int ret = 0, state; privdev = (struct drm_device*) dev->pubdev.privdev; state = PMSG_EVENT(pm_event); if (state == PMSG_EVENT(pdev->dev.power.power_state)) return 0; __KE_DEBUG("power state: %d-%d\n", state, PMSG_EVENT(pdev->dev.power.power_state)); /* lock console output to prevent kernel hang with vesafb * A temporal workaround for current kernel issue, the workaround * itself may cause a different deadlock, but it appears to * happen much less frequent then without this workaround. */ if (state == PM_EVENT_SUSPEND) acquire_console_sem(); if (firegl_cail_powerdown(privdev, state)) ret = -EIO; if (!ret) { firegl_pci_save_state(privdev); pci_disable_device(pdev); PMSG_EVENT(pdev->dev.power.power_state) = state; } else { // firegl_cail_powerdown failed. Try to restore the system to // a usable state. firegl_cail_powerup(privdev); if (state == PM_EVENT_SUSPEND) release_console_sem(); } return ret; } static int fglrx_pci_resume(struct pci_dev *pdev) { device_t* dev = &firegl_public_device; struct drm_device* privdev; privdev = (struct drm_device*) dev->pubdev.privdev; #ifdef _KE_SERIAL_DEBUG __ke_SetSerialPort(); #endif __KE_DEBUG("%d \n", PMSG_EVENT(pdev->dev.power.power_state)); if (PMSG_EVENT(pdev->dev.power.power_state) == 0) return 0; // PCI config space needs to be restored very early, in particular // before pci_set_master! firegl_pci_restore_state(privdev); if (pci_enable_device(pdev)) { __KE_ERROR("Cannot enable PCI device.\n"); } pci_set_master(pdev); firegl_cail_powerup(privdev); if (PMSG_EVENT(pdev->dev.power.power_state) == PM_EVENT_SUSPEND) release_console_sem(); PMSG_EVENT(pdev->dev.power.power_state) = 0; return 0; } static struct pci_driver fglrx_pci_driver = { .name = "fglrx_pci", .id_table = fglrx_pci_table, .probe = fglrx_pci_probe, #ifdef CONFIG_PM .suspend = fglrx_pci_suspend, .resume = fglrx_pci_resume, #endif /* CONFIG_PM */ }; #endif // FIREGL_POWER_MANAGEMENT /*****************************************************************************/ /* init_module is called when insmod is used to load the module */ static int __init firegl_init_module(void) { device_t* dev = &firegl_public_device; unsigned int i; int retcode; EXPORT_NO_SYMBOLS; // init global vars that are in fact constants __ke_HZ = HZ; #ifdef _KE_SERIAL_DEBUG __ke_SetSerialPort(); #endif // init DRM proc list drm_proclist = kmalloc((DRM_PROC_ENTRIES + 1) * sizeof(__ke_proc_list_t), GFP_KERNEL); if ( drm_proclist == NULL ) return -ENOMEM; for ( i=0; ipubdev.signature = FGL_DEVICE_SIGNATURE; for (i = 0; i < __KE_MAX_SPINLOCKS; i++) dev->spinlock[i] = SPIN_LOCK_UNLOCKED; for (i=0; i < __KE_MAX_SEMAPHORES; i++) sema_init(&dev->struct_sem[i], 1); dev->pubdev.psigmask = (__ke_sigset_t*)&dev->sigmask; if ( (retcode = firegl_init(&dev->pubdev)) ) { __KE_ERROR("firegl_init failed\n"); kfree(drm_proclist); return retcode; } #if !defined(FIREGL_IOCTL_COMPAT) && defined(__x86_64__) if(!firegl_init_32compat_ioctls()) { kfree(drm_proclist); __KE_ERROR("Couldn't register compat32 ioctls!\n"); return -ENODEV; } #endif // get the minor number firegl_minor = firegl_stub_register(dev->pubdev.name, &firegl_fops, dev); if (firegl_minor < 0) { __KE_ERROR("firegl_stub_register failed\n"); kfree(drm_proclist); return -EPERM; } dev->device = MKDEV(DRM_MAJOR, firegl_minor); __KE_INFO("module loaded - %s %d.%d.%d [%s] on minor %d\n", dev->pubdev.name, dev->pubdev.major_version, dev->pubdev.minor_version, dev->pubdev.patchlevel, dev->pubdev.date, firegl_minor); #ifdef FIREGL_POWER_MANAGEMENT if (pci_register_driver (&fglrx_pci_driver) < 0) { __KE_ERROR("Failed to register fglrx as PCI driver\n"); } #endif // FIREGL_POWER_MANAGEMENT return 0; // OK! } /* cleanup_module is called when rmmod is used to unload the module */ static void __exit firegl_cleanup_module(void) { device_t* dev = &firegl_public_device; __KE_DEBUG("module cleanup started\n"); #ifdef FIREGL_POWER_MANAGEMENT pci_unregister_driver (&fglrx_pci_driver); #endif if ( firegl_stub_unregister(firegl_minor) ) { __KE_ERROR("Cannot unload module\n"); } firegl_cleanup(&dev->pubdev); #if !defined(FIREGL_IOCTL_COMPAT) && defined(__x86_64__) firegl_kill_32compat_ioctls(); #endif if (drm_proclist) kfree(drm_proclist); __KE_INFO("module unloaded - %s %d.%d.%d [%s] on minor %d\n", dev->pubdev.name, dev->pubdev.major_version, dev->pubdev.minor_version, dev->pubdev.patchlevel, dev->pubdev.date, firegl_minor); } module_init( firegl_init_module ); module_exit( firegl_cleanup_module ); /*****************************************************************************/ // wait queues __ke_wait_queue_head_t* ATI_API_CALL __ke_alloc_wait_queue_head_struct(void) { __ke_wait_queue_head_t* queue_head; queue_head = kmalloc(sizeof(wait_queue_head_t), GFP_ATOMIC); if (queue_head) { init_waitqueue_head((wait_queue_head_t*)(void *)queue_head); } return queue_head; } void ATI_API_CALL __ke_free_wait_queue_head_struct(__ke_wait_queue_head_t* queue_head) { if (queue_head) kfree(queue_head); } __ke_wait_queue_t* ATI_API_CALL __ke_alloc_wait_queue_struct(void) { __ke_wait_queue_t* queue; queue = kmalloc(sizeof(wait_queue_t), GFP_KERNEL); return queue; } void ATI_API_CALL __ke_free_wait_queue_struct(__ke_wait_queue_t* queue) { if (queue) kfree(queue); } void ATI_API_CALL __ke_wake_up_interruptible(__ke_wait_queue_head_t* queue_head) { wake_up_interruptible((wait_queue_head_t*)(void *)queue_head); } void ATI_API_CALL __ke_add_wait_queue(__ke_wait_queue_head_t* queue_head, __ke_wait_queue_t* entry) { // initialisation (delayed) #ifdef __WAITQUEUE_INITIALIZER wait_queue_t template = __WAITQUEUE_INITIALIZER(((wait_queue_t*)(void *)entry), current); *((wait_queue_t*)(void *)entry) = template; #else ((wait_queue_t*)(void *)entry)->task = current; ((wait_queue_t*)(void *)entry)->flags = 0x0; ((wait_queue_t*)(void *)entry)->task_list.next = NULL; ((wait_queue_t*)(void *)entry)->task_list.prev = NULL; ((wait_queue_t*)(void *)entry)->func = NULL; #if WAITQUEUE_DEBUG ((wait_queue_t*)(void *)entry)->__magic = &(((wait_queue_t*)(void *)entry)->__magic); ((wait_queue_t*)(void *)entry)->__waker = 0; #endif /* WAITQUEUE_DEBUG */ #endif /* __WAITQUEUE_INITIALIZER */ // addition add_wait_queue((wait_queue_head_t*)(void *)queue_head, (wait_queue_t*)(void *)entry); } void ATI_API_CALL __ke_remove_wait_queue(__ke_wait_queue_head_t* queue_head, __ke_wait_queue_t* entry) { // current->state = TASK_RUNNING; remove_wait_queue((wait_queue_head_t*)(void *)queue_head, (wait_queue_t*)(void *)entry); } void ATI_API_CALL __ke_init_waitqueue_head(__ke_wait_queue_head_t* queue_head) { init_waitqueue_head((wait_queue_head_t*)(void *)queue_head); } void ATI_API_CALL __ke_wait_event_interruptible(__ke_wait_queue_head_t* queue_head, int condition) { wait_event_interruptible(*((wait_queue_head_t*)(void *)queue_head), condition); } void ATI_API_CALL __ke_poll_wait(struct file* filp, __ke_wait_queue_head_t* queue_head, __ke_poll_table* pt) { poll_wait(filp, (wait_queue_head_t*)(void*)queue_head, (poll_table*)(void*)pt); } // scheduler void ATI_API_CALL __ke_schedule(void) { schedule(); } /** /brief This routine will let the current processor yield for other threads. */ void ATI_API_CALL __ke_yield(void) { yield(); } int ATI_API_CALL __ke_signal_pending(void) { return signal_pending(current); } void ATI_API_CALL __ke_set_current_state_task_interruptible(void) { current->state = TASK_INTERRUPTIBLE; } void ATI_API_CALL __ke_set_current_state_task_running(void) { current->state = TASK_RUNNING; } void ATI_API_CALL __ke_configure_sigmask(__ke_sigset_t *pSigMask) { sigemptyset((sigset_t*)(void *)pSigMask); sigaddset((sigset_t*)(void *)pSigMask, SIGSTOP); sigaddset((sigset_t*)(void *)pSigMask, SIGTSTP); sigaddset((sigset_t*)(void *)pSigMask, SIGTTIN); sigaddset((sigset_t*)(void *)pSigMask, SIGTTOU); } int firegl_sig_notifier_wrap(void *priv) { return firegl_sig_notifier(priv); } void ATI_API_CALL __ke_block_all_signals(int (*notifier)(void *priv), void *pPriv, __ke_sigset_t *pSigMask) { block_all_signals(notifier,pPriv,(sigset_t*)(void *)pSigMask); } void ATI_API_CALL __ke_unblock_all_signals(void) { unblock_all_signals(); } #if defined(__i386__) #ifndef __HAVE_ARCH_CMPXCHG static inline unsigned long __fgl_cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { unsigned long prev; switch (size) { case 1: __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" : "=a"(prev) : "q"(new), "m"(*__xg(ptr)), "0"(old) : "memory"); return prev; case 2: __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" : "=a"(prev) : "q"(new), "m"(*__xg(ptr)), "0"(old) : "memory"); return prev; case 4: __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" : "=a"(prev) : "q"(new), "m"(*__xg(ptr)), "0"(old) : "memory"); return prev; } return old; } #endif /* cmpxchg */ #elif defined(__alpha__) todo !!! #endif #if !defined(__ia64__) unsigned long ATI_API_CALL __ke__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { #ifndef __HAVE_ARCH_CMPXCHG return __fgl_cmpxchg(ptr,old,new,size); #else return __cmpxchg(ptr,old,new,size); #endif } #endif /*****************************************************************************/ __ke_dev_t ATI_API_CALL __ke_getdevice(__ke_device_t *dev) { return ((device_t*)dev)->device; } const char* ATI_API_CALL __ke_module_parm(void) { return firegl; } /*****************************************************************************/ int ATI_API_CALL __ke_inode_rdev_minor(struct inode* inode) { #ifndef MINOR return minor(inode->i_rdev); #else return MINOR(inode->i_rdev); #endif } /*****************************************************************************/ void* ATI_API_CALL __ke_get_proc_dir_entry_priv(struct proc_dir_entry *pde) { return pde->data; } void* ATI_API_CALL __ke_get_inode_priv(struct inode* inode) { #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,4) return inode->u.generic_ip; #else return PDE(inode); #endif } void* ATI_API_CALL __ke_get_file_priv(struct file* filp) { return filp->private_data; } void ATI_API_CALL __ke_set_file_priv(struct file* filp, void* private_data) { filp->private_data = private_data; } int ATI_API_CALL __ke_file_excl_open(struct file* filp) { return (filp->f_flags & O_EXCL) != 0; } int ATI_API_CALL __ke_file_rw_open(struct file* filp) { return (filp->f_flags & 3) != 0; } unsigned int ATI_API_CALL __ke_file_counter(struct file *filp) { return filp->f_count.counter; } struct inode* ATI_API_CALL __ke_get_file_inode(struct file* filp) { return (struct inode*)filp->f_dentry->d_inode; } /*****************************************************************************/ int ATI_API_CALL __ke_getpid(void) { return current->pid; } int ATI_API_CALL __ke_geteuid(void) { return current->euid; } /*****************************************************************************/ unsigned long ATI_API_CALL __ke_jiffies(void) { return jiffies; } void ATI_API_CALL __ke_udelay(unsigned long usecs) // delay in usec { unsigned long start; unsigned long stop; unsigned long period; unsigned long wait_period; struct timespec tval; #ifdef NDELAY_LIMIT // kernel provides delays with nano(=n) second accuracy #define UDELAY_LIMIT (NDELAY_LIMIT/1000) /* supposed to be 10 msec */ #else // kernel provides delays with micro(=u) second accuracy #define UDELAY_LIMIT (10000) /* 10 msec */ #endif if (usecs > UDELAY_LIMIT) { start = jiffies; tval.tv_sec = usecs / 1000000; tval.tv_nsec = (usecs - tval.tv_sec * 1000000) * 1000; wait_period = timespec_to_jiffies(&tval); do { stop = jiffies; if (stop < start) // jiffies overflow period = ((unsigned long)-1 - start) + stop + 1; else period = stop - start; } while (period < wait_period); } else udelay(usecs); /* delay value might get checked once again */ } void ATI_API_CALL __ke_mdelay(unsigned long msecs) // delay in msec { mdelay(msecs); } __ke_u8 ATI_API_CALL __ke_readb(void* p, __ke_u32 offset) { return readb((u8*)p + offset); } /*****************************************************************************/ // TODO: These here get obsolete in future, use the ia64 code below // Johannes unsigned long ATI_API_CALL __ke_virt_to_bus(void* address) { return virt_to_bus(address); } unsigned long ATI_API_CALL __ke_virt_to_phys(void* address) { return virt_to_phys(address); } void* ATI_API_CALL __ke_high_memory(void) { return high_memory; } int ATI_API_CALL __ke_config_x86_4G(void) { #ifdef CONFIG_X86_4G return 1; #else return 0; #endif } int ATI_API_CALL __ke_pci_enable_device(__ke_pci_dev_t* dev) { return (pci_enable_device( (struct pci_dev*)(void *)dev )); } #if defined(__x86_64__) || defined(__ia64__) void* ATI_API_CALL __ke_pci_alloc_consistent(__ke_pci_dev_t* dev, int size, void *dma_handle) { return (pci_alloc_consistent( (struct pci_dev*)(void *)dev, size, dma_handle)); } void ATI_API_CALL __ke_pci_free_consistent(__ke_pci_dev_t* dev, int size, unsigned long cpu_addr, unsigned int dma_handle) { pci_free_consistent( (struct pci_dev*)(void *)dev, size, (void *)cpu_addr, (unsigned long)dma_handle); } #endif // __ia64__ /*****************************************************************************/ int ATI_API_CALL __ke_error_code(enum __ke_error_num errcode) { switch (errcode) { case __KE_EBUSY: return EBUSY; case __KE_EINVAL: return EINVAL; case __KE_EACCES: return EACCES; case __KE_EFAULT: return EFAULT; case __KE_EIO: return EIO; case __KE_EBADSLT: return EBADSLT; case __KE_ENOMEM: return ENOMEM; case __KE_EPERM: return EPERM; case __KE_ENODEV: return ENODEV; case __KE_EINTR: return EINTR; case __KE_ERESTARTSYS: return ERESTARTSYS; case __KE_ELIBBAD: return ELIBBAD; default: return EFAULT; } } /*****************************************************************************/ #ifdef __x86_64__ int ATI_API_CALL firegl_get_user_ptr(u32 *src, void **dst) { unsigned long temp; int err = get_user(temp, src); *dst = (void*) temp; return err; } int ATI_API_CALL firegl_get_user_u16(u16 *src, u16 *dst) { u16 temp; int err = get_user(temp, src); *dst = temp; return err; } int ATI_API_CALL firegl_get_user_u32(u32 *src, u32 *dst) { u32 temp; int err = get_user(temp, src); *dst = temp; return err; } int ATI_API_CALL firegl_get_user_u64(u32 *src, u64 *dst) { u64 temp; int err = get_user(temp, src); *dst = temp; return err; } int ATI_API_CALL firegl_put_user_ptr(void *src, u32 *dst) { void *temp = src; return put_user(temp, dst); } int ATI_API_CALL firegl_put_user_u16(u16 src, u16 *dst) { u16 temp = src; return put_user(temp, dst); } int ATI_API_CALL firegl_put_user_u32(u32 src, u32 *dst) { u32 temp = src; return put_user(temp, dst); } int ATI_API_CALL firegl_put_user_u64(u64 src, u32 *dst) { u64 temp = src; return put_user(temp, dst); } #endif /* __x86_64__ */ /*****************************************************************************/ void ATI_API_CALL __ke_mod_inc_use_count(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) __module_get(THIS_MODULE); #else MOD_INC_USE_COUNT; #endif } void ATI_API_CALL __ke_mod_dec_use_count(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) module_put(THIS_MODULE); #else MOD_DEC_USE_COUNT; #endif } /*****************************************************************************/ void ATI_API_CALL __ke_down_struct_sem(__ke_device_t *dev, int index) { down(&(((device_t*)dev)->struct_sem[index])); } void ATI_API_CALL __ke_up_struct_sem(__ke_device_t *dev, int index) { up(&(((device_t*)dev)->struct_sem[index])); } void ATI_API_CALL __ke_sema_init(struct semaphore* sem, int value) { sema_init(sem, value); } __ke_size_t ATI_API_CALL __ke_sema_size(void) { return sizeof(struct semaphore); } void ATI_API_CALL __ke_down(struct semaphore* sem) { down(sem); } void ATI_API_CALL __ke_up(struct semaphore* sem) { up(sem); } /*****************************************************************************/ void ATI_API_CALL __ke_atomic_inc(void* v) { atomic_inc((atomic_t*)v); } void ATI_API_CALL __ke_atomic_dec(void* v) { atomic_dec((atomic_t*)v); } void ATI_API_CALL __ke_atomic_add(int val, void* v) { atomic_add(val, (atomic_t*)v); } void ATI_API_CALL __ke_atomic_sub(int val, void* v) { atomic_sub(val, (atomic_t*)v); } int ATI_API_CALL __ke_atomic_read(void* v) { return atomic_read((atomic_t*)v); } void ATI_API_CALL __ke_atomic_set(void* v, int val) { atomic_set((atomic_t*)v, val); } int ATI_API_CALL __ke_atomic_dec_and_test(void* v) { return atomic_dec_and_test((atomic_t*)v); } /*****************************************************************************/ void ATI_API_CALL __ke_spin_lock(__ke_device_t *dev, int ndx) { spin_lock(&(((device_t*)dev)->spinlock[ndx])); } void ATI_API_CALL __ke_spin_unlock(__ke_device_t *dev __attribute__((unused)), int ndx __attribute__((unused))) { spin_unlock(&(((device_t*)dev)->spinlock[ndx])); } void ATI_API_CALL __ke_lock_kernel(void) { lock_kernel(); } void ATI_API_CALL __ke_unlock_kernel(void) { unlock_kernel(); } /*****************************************************************************/ #ifdef FGL_USE_SCT extern unsigned long sys_call_table[]; #endif typedef int (*PFNMLOCK)(unsigned long start, __ke_size_t len); typedef int (*PFNMUNLOCK)(unsigned long start, __ke_size_t len); int ATI_API_CALL __ke_sys_mlock(unsigned long start, __ke_size_t len) { #ifdef FGL_USE_SCT PFNMLOCK sys_mlock = (PFNMLOCK)sys_call_table[__NR_mlock]; if (!sys_mlock) { __KE_ERROR("sys_call_table[__NR_mlock] == 0\n"); return -1; } return (*sys_mlock)(start, len); #else return mlock((void*)start, len); #endif } int ATI_API_CALL __ke_sys_munlock(unsigned long start, __ke_size_t len) { #ifdef FGL_USE_SCT PFNMUNLOCK sys_munlock = (PFNMUNLOCK)sys_call_table[__NR_munlock]; if (!sys_munlock) { __KE_ERROR("sys_call_table[__NR_munlock] == 0\n"); return -1; } return (*sys_munlock)(start, len); #else return munlock((void*)start, len); #endif } typedef int (*PFNMODIFYLDT)(int func, void *ptr, unsigned long bytecount); int ATI_API_CALL __ke_sys_modify_ldt(int func, void *ptr, unsigned long bytecount) { #ifdef FGL_USE_SCT PFNMODIFYLDT sys_modify_ldt = (PFNMODIFYLDT)sys_call_table[__NR_modify_ldt]; if (!sys_modify_ldt) { __KE_ERROR("sys_call_table[__NR_modify_ldt] == 0\n"); return -1; } return (*sys_modify_ldt)(func, ptr, bytecount); #else #if !defined(__ia64__) && !defined(__x86_64__) return modify_ldt(func, ptr, bytecount); #else // TODO: how should this be down on ia64???? return 0; #endif #endif } /*****************************************************************************/ int ATI_API_CALL __ke_vsprintf(char *buf, const char *fmt, va_list ap) { return vsprintf(buf, fmt, ap); } int ATI_API_CALL __ke_vsnprintf(char *buf, size_t size, const char *fmt, va_list ap) { return vsnprintf(buf, size, fmt, ap); } #ifdef __KE_NO_VSPRINTF void ATI_API_CALL __ke_printk(const char* fmt, ...) { char buffer[256]; va_list marker; va_start(marker, fmt); vsprintf(buffer, fmt, marker); va_end(marker); printk(buffer); #ifdef _KE_SERIAL_DEBUG __ke_SerPrint(buffer); #endif } #else void ATI_API_CALL __ke_print_info(const char* fmt, ...) { char msg[256] = KERN_INFO;: va_list marker; va_start(marker, fmt); vsprintf(msg + strlen(msg), fmt, marker); va_end(marker); } void ATI_API_CALL __ke_print_error(const char* fmt, ...) { char msg[256] = KERN_ERR; va_list marker; va_start(marker, fmt); vsprintf(msg + strlen(msg), fmt, marker); va_end(marker); } void ATI_API_CALL __ke_print_debug(const char* fmt, ...) { char msg[256] = KERN_DEBUG; va_list marker; va_start(marker, fmt); vsprintf(msg + strlen(msg), fmt, marker); va_end(marker); } #endif #ifdef _KE_SERIAL_DEBUG // To enable serial port debug message dumping,just define _KE_SERIAL_DEBUG in firegl_public.h file. // Connect two PC with a null modern serial cable. run Hyper ternimal on the remote machine. // It's useful to debug resume if network not works properly and serial port is not recovered // properly when fglrx resume hook is called... #define SER_DATA_PORT 0x3f8 #define SER_INT_CTRL_PORT SER_DATA_PORT + 1 #define SER_INT_STAT_PORT SER_DATA_PORT + 2 #define SER_LINE_CTRL_PORT SER_DATA_PORT + 3 #define SER_MODEM_CTRL_PORT SER_DATA_PORT + 4 #define SER_LINE_STAT_PORT SER_DATA_PORT + 5 void ATI_API_CALL __ke_printc(char c) { while((inb(SER_LINE_STAT_PORT) & 0x20) == 0 ); //wait until Transmitter Holding Register Empty outb(c,SER_DATA_PORT); } void ATI_API_CALL __ke_printstr(const char *str) { int len = strlen(str); while(len--)__ke_printc(*str++); } int ATI_API_CALL __ke_SerPrint(const char *format, ...) { char buffer[256]; va_list ap; va_start(ap, format); vsprintf(buffer, format, ap); va_end(ap); __ke_printstr(buffer); return 0; } void ATI_API_CALL __ke_SetSerialPort() { DRM_INFO("setup serial port\n"); outb(0x00, SER_INT_CTRL_PORT); // Turn off interrupts outb(0x80, SER_LINE_CTRL_PORT); // SET DLAB ON outb(0x01, SER_DATA_PORT); // Set Baud rate - Divisor Latch Low Byte // 0x01 = 115,200 ,0x02 = 57,600, 0x06 = 19,200 BPS, 0x0C = 9,600 BPS outb(0x00, SER_DATA_PORT + 1); // Set Baud rate - Divisor Latch High Byte outb(0x03, SER_LINE_CTRL_PORT); // reset DLAB ,8 Bits, No Parity, 1 Stop Bit outb(0xC7, SER_DATA_PORT + 2); // FIFO Control Register outb(0x0b, SER_DATA_PORT + 4); // Turn on DTR, RTS, and OUT2 __ke_printstr("serial port 0x3f8 is set ready for message print out \n"); } #endif /*****************************************************************************/ int ATI_API_CALL __ke_capable(enum __ke_cap cap) { switch (cap) { case __KE_CAP_SYS_ADMIN: cap = CAP_SYS_ADMIN; break; case __KE_CAP_IPC_LOCK: cap = CAP_IPC_LOCK; break; default: return 0; } return capable(cap); } void ATI_API_CALL __ke_cap_effective_raise(enum __ke_cap cap) { switch (cap) { case __KE_CAP_SYS_ADMIN: cap = CAP_SYS_ADMIN; break; case __KE_CAP_IPC_LOCK: cap = CAP_IPC_LOCK; break; default: return; } cap_raise(current->cap_effective, cap); } __ke_u32 ATI_API_CALL __ke_get_cap_effective() { return cap_t(current->cap_effective); } void ATI_API_CALL __ke_set_cap_effective(__ke_u32 cap) { cap_t(current->cap_effective) = cap; } unsigned long ATI_API_CALL __ke_ram_available(void) { struct sysinfo si; si_meminfo(&si); #if LINUX_VERSION_CODE < 0x020317 /* Changed to page count in 2.3.23 */ return si.totalram >> PAGE_SHIFT; #else return si.totalram; #endif } #ifdef __x86_64__ void* ATI_API_CALL __ke_compat_alloc_user_space(long size) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) return compat_alloc_user_space(size); #else struct pt_regs *regs = (void *)current->thread.rsp0 - sizeof(struct pt_regs); return (void __user *)regs->rsp - size; #endif } #endif int ATI_API_CALL __ke_copy_from_user(void* to, const void* from, __ke_size_t size) { return copy_from_user(to, from, size); } int ATI_API_CALL __ke_copy_to_user(void* to, const void* from, __ke_size_t size) { return copy_to_user(to, from, size); } int ATI_API_CALL __ke_verify_area(int type, const void * addr, unsigned long size) { #ifdef access_ok return access_ok(type,addr,size) ? 0 : -EFAULT; #else return verify_area(type, addr, size); #endif } int ATI_API_CALL __ke_get_pci_device_info(__ke_pci_dev_t* dev, __ke_pci_device_info_t *pinfo) { if ( dev ) { pinfo->vendor = ((struct pci_dev*)(void *)dev)->vendor; pinfo->device = ((struct pci_dev*)(void *)dev)->device; pinfo->subsystem_vendor = ((struct pci_dev*)(void *)dev)->subsystem_vendor; pinfo->subsystem_device = ((struct pci_dev*)(void *)dev)->subsystem_device; } return -EINVAL; } int ATI_API_CALL __ke_check_pci(int busnum, int devnum, int funcnum, __ke_u16* vendor, __ke_u16* device, unsigned int* irq) { struct pci_dev* pci_dev; pci_dev = pci_find_slot(busnum, PCI_DEVFN(devnum, funcnum)); if (!pci_dev) return 0; if (vendor) *vendor = pci_dev->vendor; if (device) *device = pci_dev->device; if (irq) *irq = pci_dev->irq; return 1; } int ATI_API_CALL __ke_pci_get_irq(__ke_pci_dev_t *dev, unsigned int* irq) { if (!dev) return 0; if (!irq) return 0; *irq = ((struct pci_dev*)dev)->irq; return 1; } __ke_pci_dev_t* ATI_API_CALL __ke_pci_find_device (unsigned int vendor, unsigned int dev, __ke_pci_dev_t* from) { return (__ke_pci_dev_t*)pci_find_device( vendor, dev, (struct pci_dev *)(void *)from ); } void* ATI_API_CALL __ke_malloc(__ke_size_t size) { return kmalloc(size, GFP_KERNEL); } void* ATI_API_CALL __ke_malloc_atomic(__ke_size_t size) { return kmalloc(size, GFP_ATOMIC); } void ATI_API_CALL __ke_free(void* p) { kfree(p); } void ATI_API_CALL __ke_free_s(void* p, __ke_size_t size) { kfree(p); } void* ATI_API_CALL __ke_vmalloc(__ke_size_t size) { return vmalloc(size); } void* ATI_API_CALL __ke_vmalloc_32(__ke_size_t size) { return vmalloc_32(size); } void* ATI_API_CALL __ke_vmalloc_atomic(__ke_size_t size) { return __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); } void ATI_API_CALL __ke_vfree(void* p) { return vfree(p); } void* ATI_API_CALL __ke_get_free_page(void) { return (void*)__get_free_page(GFP_KERNEL); } void* ATI_API_CALL __ke_get_free_pages(int order) { return (void*)__get_free_pages(GFP_KERNEL|__GFP_COMP, order); } void ATI_API_CALL __ke_free_page(void* pt) { free_page((unsigned long)pt); } void ATI_API_CALL __ke_free_pages(void* pt, int order) { free_pages((unsigned long)pt, order); } void ATI_API_CALL __ke_get_page(void* pt) { get_page(virt_to_page((unsigned long)pt)); } void ATI_API_CALL __ke_put_page(void* pt) { put_page(virt_to_page((unsigned long)pt)); } void ATI_API_CALL __ke_unlock_page(void *virt) { unlock_page(virt_to_page((unsigned long)virt)); } int ATI_API_CALL __ke_PageCompound(void *virt) { #ifdef PageCompound return PageCompound(virt_to_page((unsigned long)virt)); #else return 0; #endif } void *ATI_API_CALL __ke_vmalloc_to_addr(void *vmalloc_addr) { struct page *page = NULL; page = vmalloc_to_page(vmalloc_addr); if(page == NULL) { __KE_ERROR("__ke_vmalloc_to_addr: invalid page!"); return NULL; } return page_address(page); } unsigned long ATI_API_CALL __ke_do_mmap(struct file * file, unsigned long addr, unsigned long len, unsigned long pgoff) { unsigned long flags, prot; unsigned long vaddr; flags = MAP_SHARED; prot = PROT_READ|PROT_WRITE; down_write(¤t->mm->mmap_sem); vaddr = do_mmap(file, 0, len, prot, flags, pgoff); up_write(¤t->mm->mmap_sem); return addr; } int ATI_API_CALL __ke_do_munmap(unsigned long addr, unsigned long len) { int retcode = 0; down_write(¤t->mm->mmap_sem); #ifdef FGL_LINUX_RHEL_MUNMAP_API retcode = do_munmap(current->mm, addr, len, 1); #else retcode = do_munmap(current->mm, addr, len); #endif up_write(¤t->mm->mmap_sem); return retcode; } #if defined(__i386__) /* * The implementation of these 64bit arithmetic functions is a clean re-implementation * after observing FreeBSD libkern v1.6 code. */ #ifdef do_div unsigned long long __udivdi3(unsigned long long n, unsigned long long base) { do_div(n, base); return n; } unsigned long long __umoddi3(unsigned long long n, unsigned long long base) { return do_div(n, base); } long long __divdi3(long long n, long long base) { unsigned long long un, ubase; int minus = 0; if (n < 0) { un = -n; minus = 1; } else { un = n; minus = 0; } if (base < 0) { ubase = -base; minus = !minus; } else { ubase = base; } do_div(un, ubase); return (minus? -un : un); } long long __moddi3(long long n, long long base) { unsigned long long un, ubase; unsigned long long rem; int minus = 0; if (n < 0) { un = -n; minus = 1; } else { un = n; minus = 0; } if (base < 0) { ubase = -base; minus = !minus; } else { ubase = base; } rem = do_div(un, ubase); return (minus? -rem : rem); } #endif #endif #if defined(VM_MAP) || defined(vunmap) void* ATI_API_CALL __ke_vmap(unsigned long *pagelist, unsigned int count) { struct page * pages[count]; int i; for (i=0; iflags); } void ATI_API_CALL __ke_virt_unreserve(void* virt) { clear_bit(PG_reserved, &virt_to_page((unsigned long)virt)->flags); } void ATI_API_CALL __ke_lock_page(void* virt) { lock_page(virt_to_page((unsigned long)virt)); } #ifdef __ia64__ void* ATI_API_CALL __ke_get_vmptr( struct _agp_memory* memory ) { return memory->vmptr; } #endif void* ATI_API_CALL __ke_ioremap(unsigned long offset, unsigned long size) { return ioremap(offset, size); } void* ATI_API_CALL __ke_ioremap_nocache(unsigned long offset, unsigned long size) { return ioremap_nocache(offset, size); } void ATI_API_CALL __ke_iounmap(void* pt) { iounmap(pt); } int ATI_API_CALL __ke_verify_read_access(void* addr, __ke_size_t size) { return access_ok(VERIFY_READ, addr, size) ? 0 : -EFAULT; } int ATI_API_CALL __ke_verify_write_access(void* addr, __ke_size_t size) { return access_ok(VERIFY_WRITE, addr, size) ? 0 : -EFAULT; } struct mm_struct* ATI_API_CALL __ke_init_mm(void) { return &init_mm; } struct mm_struct* ATI_API_CALL __ke_current_mm(void) { return current->mm; } __ke_dma_addr_t ATI_API_CALL __ke_get_vm_phys_addr(struct mm_struct* mm, unsigned long virtual_addr) { pgd_t* pgd_p; pmd_t* pmd_p; pte_t pte; PGD_OFFSET(mm, pgd_p, virtual_addr); PGD_PRESENT(pgd_p); PMD_OFFSET(pmd_p, pgd_p, virtual_addr); PMD_PRESENT(pmd_p); PTE_OFFSET(pte, pmd_p, virtual_addr); PTE_PRESENT(pte); #if defined(__x86_64__) || defined(__ia64__) return pte.pte & PAGE_MASK; #else return pte_val(pte) & (u64)((u64)PAGE_MASK | (u64)0xf<<32); #endif } unsigned long ATI_API_CALL __ke_get_vm_page_table(struct mm_struct* mm, unsigned long virtual_addr, unsigned long* page_addr, int* is_highpte) { pgd_t* pgd_p; pmd_t* pmd_p; unsigned long page_table; __KE_DEBUG("virtual_addr=0x%08lx\n", virtual_addr); PGD_OFFSET(mm, pgd_p, virtual_addr); PGD_PRESENT(pgd_p); __KE_DEBUG("pgd_p=0x%08lx\n", (unsigned long)pgd_p); PMD_OFFSET(pmd_p, pgd_p, virtual_addr); PMD_PRESENT(pmd_p); __KE_DEBUG("pmd_p=0x%08lx\n", (unsigned long)pmd_p); *page_addr = (unsigned long) PMD_PAGE(*pmd_p); if (PageHighMem((struct page*)(*page_addr))) { page_table = (unsigned long) kmap((struct page*)(*page_addr)); *is_highpte = 1; } else { page_table = (unsigned long)phys_to_virt(pmd_val(*pmd_p) & PAGE_MASK); *is_highpte = 0; } __KE_DEBUG("page_table %lx, page_addr %lx, is_highpte %d\n", page_table, *page_addr, *is_highpte); return page_table; } void ATI_API_CALL __ke_put_vm_page_table(unsigned long page_addr) { kunmap((struct page*) page_addr); } #ifndef ptep_clear_flush_dirty #define ptep_clear_flush_dirty(__vma, __address, __ptep) \ ({ \ int __dirty = ptep_test_and_clear_dirty(__ptep); \ if (__dirty) \ flush_tlb_page(__vma, __address); \ __dirty; \ }) #endif int ATI_API_CALL __ke_vm_test_and_clear_dirty(struct mm_struct* mm, unsigned long virtual_addr) { int ret = -1; // init with page not present pgd_t* pgd_p; pmd_t* pmd_p; pte_t* pte_p; pte_t pte; struct vm_area_struct *vma; __KE_DEBUG("virtual_addr=0x%08lx\n", virtual_addr); vma = find_vma(mm, virtual_addr);; if (NULL == vma) { __KE_DEBUG("%s", "ERROR: find_vma failed\n"); return -1; } PGD_OFFSET(mm, pgd_p, virtual_addr); if (!pgd_present(*pgd_p)) { __KE_DEBUG("ERROR: !pgd_present\n"); return -1; } __KE_DEBUG("pgd_p=0x%08lx\n", (unsigned long)pgd_p); PMD_OFFSET(pmd_p, pgd_p, virtual_addr); if (!pmd_present(*pmd_p)) { __KE_DEBUG("ERROR: !pmd_present\n"); return -1; } __KE_DEBUG("pmd_p=0x%08lx\n", (unsigned long)pmd_p); #ifdef pte_offset_atomic pte_p = pte_offset_atomic(pmd_p, virtual_addr); if (pte_present(*pte_p)) ret = (ptep_clear_flush_dirty(vma, virtual_addr, pte_p) ? 1 : 0); else __KE_DEBUG("page not exists!\n"); pte_kunmap(pte_p); #else #ifdef pte_offset_map pte_p = pte_offset_map(pmd_p, virtual_addr); if (pte_present(*pte_p)) ret = (ptep_clear_flush_dirty(vma, virtual_addr, pte_p) ? 1 : 0); else __KE_DEBUG("page not exists!\n"); pte_unmap(pte_p); #else #ifdef pte_offset_kernel pte_p = pte_offset_kernel(pmd_p, virtual_addr); if (pte_present(*pte_p)) ret = (ptep_clear_flush_dirty(vma, virtual_addr, pte_p) ? 1 : 0); else __KE_DEBUG("page not exists!\n"); #else pte_p = pte_offset(pmd_p, virtual_addr); if (pte_present(*pte_p)) ret = (ptep_clear_flush_dirty(vma, virtual_addr, pte_p) ? 1 : 0); else __KE_DEBUG("page not exists!\n"); #endif #endif #endif if (debuglevel > 2) { char buf[50]; __ke_dma_addr_t page_phys_addr; __KE_DEBUG("pte: %s\n", __ke_pte_phys_addr_str(pte, buf, &page_phys_addr)); } __KE_DEBUG("return %d\n", ret); return ret; } __ke_dma_addr_t* ATI_API_CALL __ke_get_vm_phys_addr_list(struct mm_struct* mm, unsigned long virtual_addr, unsigned long pages) { __ke_dma_addr_t *phys_table, *pt; unsigned long n, va; /* caller is responsible for the respective kfree call */ phys_table = kmalloc(pages*sizeof(__ke_dma_addr_t), GFP_KERNEL); if (!phys_table) return NULL; pt = phys_table; va = virtual_addr; for(n=0; n 0) barrier(); } #endif /* __SMP__ */ int ATI_API_CALL __ke_flush_cache(void) { #ifdef __SMP__ #if LINUX_VERSION_CODE < 0x020501 atomic_set(&cpus_waiting, smp_num_cpus - 1); #endif /* write back invalidate all other CPUs (exported by kernel) */ if (smp_call_function(deferred_flush, NULL, 1, 0) != 0) panic("timed out waiting for the other CPUs!\n"); /* invalidate this CPU */ #if defined(__i386__) || defined(__x86_64__) asm volatile ("wbinvd":::"memory"); #elif defined(__alpha__) || defined(__ia64__) || defined(__sparc__) mb(); #else #error "Please define flush_cache for your architecture." #endif while (atomic_read(&cpus_waiting) > 0) barrier(); #else /* !__SMP__ */ #if defined(__i386__) || defined(__x86_64__) asm volatile ("wbinvd":::"memory"); #elif defined(__alpha__) || defined(__ia64__) || defined(__sparc__) mb(); #else #error "Please define flush_cache for your architecture." #endif #endif /* !__SMP__ */ return 0; } /*****************************************************************************/ int ATI_API_CALL __ke_config_mtrr(void) { #ifdef CONFIG_MTRR return 1; #else /* !CONFIG_MTRR */ return 0; #endif /* !CONFIG_MTRR */ } int ATI_API_CALL __ke_mtrr_add_wc(unsigned long base, unsigned long size) { #ifdef CONFIG_MTRR return mtrr_add(base, size, MTRR_TYPE_WRCOMB, 1); #else /* !CONFIG_MTRR */ return -EPERM; #endif /* !CONFIG_MTRR */ } int ATI_API_CALL __ke_mtrr_del(int reg, unsigned long base, unsigned long size) { #ifdef CONFIG_MTRR return mtrr_del(reg, base, size); #else /* !CONFIG_MTRR */ return -EPERM; #endif /* !CONFIG_MTRR */ } int ATI_API_CALL __ke_has_vmap(void) { // We disable vmap for 2.4.x kernel to work around the big memory( > 4GB ) issue. #if LINUX_VERSION_CODE < 0x020501 return 0; #else #if defined(VM_MAP) || defined(vunmap) return 1; #else return 0; #endif #endif } #ifdef __x86_64__ int ATI_API_CALL __ke_config_iommu(void) { #ifdef CONFIG_GART_IOMMU return 1; #else /* !CONFIG_GART_IOMMU */ return 0; #endif /* !CONFIG_GART_IOMMU */ } int ATI_API_CALL __ke_no_iommu(void) { return 0; } #endif /*****************************************************************************/ int ATI_API_CALL __ke_pci_read_config_byte(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u8 *val) { return pci_read_config_byte((struct pci_dev*)(void *)dev, where, val); } int ATI_API_CALL __ke_pci_read_config_word(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u16 *val) { return pci_read_config_word((struct pci_dev*)(void *)dev, where, val); } int ATI_API_CALL __ke_pci_read_config_dword(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u32 *val) { return pci_read_config_dword((struct pci_dev*)(void *)dev, where, val); } int ATI_API_CALL __ke_pci_write_config_byte(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u8 val) { return pci_write_config_byte((struct pci_dev*)(void *)dev, where, val); } int ATI_API_CALL __ke_pci_write_config_word(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u16 val) { return pci_write_config_word((struct pci_dev*)(void *)dev, where, val); } int ATI_API_CALL __ke_pci_write_config_dword(__ke_pci_dev_t* dev, __ke_u8 where, __ke_u32 val) { return pci_write_config_dword((struct pci_dev*)(void *)dev, where, val); } __ke_pci_dev_t* ATI_API_CALL __ke_pci_find_slot(__ke_u32 bus, __ke_u32 slot) { return (__ke_pci_dev_t*)pci_find_slot(bus, slot); } __ke_u8 ATI_API_CALL __ke_pci_get_busnr(__ke_pci_dev_t* pcidev) { struct pci_dev* dev = (struct pci_dev*)pcidev; return dev->bus->number; } __ke_dma_addr_t ATI_API_CALL __ke_pci_map_single (__ke_pci_dev_t *pdev, void *buffer, __ke_size_t size, int direction) { return pci_map_single((struct pci_dev*)(void*)pdev, buffer, size, direction); } void ATI_API_CALL __ke_pci_unmap_single (__ke_pci_dev_t *pdev, __ke_dma_addr_t bus_addr, __ke_size_t size, int direction) { pci_unmap_single((struct pci_dev*)(void*)pdev, bus_addr, size, direction); } __ke_dma_addr_t ATI_API_CALL __ke_pci_map_page (__ke_pci_dev_t *pdev, unsigned long buffer, unsigned long offset, __ke_size_t size, int direction) { return pci_map_page((struct pci_dev*)(void*)pdev, virt_to_page(buffer), offset, size, direction); } void ATI_API_CALL __ke_pci_unmap_page (__ke_pci_dev_t *pdev, __ke_dma_addr_t bus_addr, __ke_size_t size, int direction) { pci_unmap_page((struct pci_dev*)(void*)pdev, bus_addr, size, direction); } /*****************************************************************************/ void ATI_API_CALL __ke_outb(unsigned char value, unsigned short port) { outb(value, port); } void ATI_API_CALL __ke_outw(unsigned short value, unsigned short port) { outw(value, port); } void ATI_API_CALL __ke_outl(unsigned int value, unsigned short port) { outl(value, port); } char ATI_API_CALL __ke_inb(unsigned short port) { return inb(port); } short ATI_API_CALL __ke_inw(unsigned short port) { return inw(port); } int ATI_API_CALL __ke_inl(unsigned short port) { return inl(port); } int ATI_API_CALL __ke_log2(unsigned long x) { return ffz(~(x)); } /*****************************************************************************/ // Interrupt support void ATI_API_CALL __ke_enable_irq(int irq) { enable_irq( irq ); } void ATI_API_CALL __ke_disable_irq(int irq) { disable_irq( irq ); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,71) int ATI_API_CALL __ke_request_irq(unsigned int irq, void (*ATI_API_CALL handler)(int, void *, void *), const char *dev_name, void *dev_id) { return request_irq(irq, (void(*)(int, void *, struct pt_regs *))handler, SA_SHIRQ, dev_name, dev_id); } void ATI_API_CALL __ke_free_irq(unsigned int irq, void *dev_id) { free_irq(irq, dev_id); } #else static void ATI_API_CALL (*irq_handler_func)(int, void*, void*); /* function pointer variable */ static irqreturn_t ke_irq_handler_wrap(int irq, void *arg1, struct pt_regs *regs) { irq_handler_func(irq, arg1, regs); return IRQ_HANDLED; } int ATI_API_CALL __ke_request_irq(unsigned int irq, void (*ATI_API_CALL handler)(int, void *, void *), const char *dev_name, void *dev_id) { irq_handler_func = handler; return request_irq(irq, ke_irq_handler_wrap, SA_SHIRQ, dev_name, dev_id); } void ATI_API_CALL __ke_free_irq(unsigned int irq, void *dev_id) { free_irq(irq, dev_id); irq_handler_func = NULL; } #endif #ifdef __x86_64__ int ATI_API_CALL __ke_register_ioctl32_conversion(unsigned int cmd, int (*handler)(unsigned int, unsigned int, unsigned long, struct file*)) { #ifdef FIREGL_IOCTL_COMPAT return 0; #else return register_ioctl32_conversion(cmd, handler); #endif } void ATI_API_CALL __ke_unregister_ioctl32_conversion(unsigned int cmd) { #ifdef FIREGL_IOCTL_COMPAT return 0; #else unregister_ioctl32_conversion(cmd); #endif } #endif /* agp_memory related routine for IGP */ int ATI_API_CALL __ke_agp_memory_get_page_count(struct _agp_memory* agpmem) { return (int)(agpmem->page_count); } void ATI_API_CALL __ke_agp_memory_get_memory(struct _agp_memory* agpmem, unsigned long **memory_ptr) { __KE_DEBUG("[%s] agpmem=0x%016lx agpmem->memory=0x%016lx [0]=0x%016x", __FUNCTION__, (unsigned long)agpmem, (unsigned long)agpmem->memory, (agpmem->memory)[0]); *memory_ptr = agpmem->memory; } /*****************************************************************************/ #ifndef NOPAGE_SIGBUS #define NOPAGE_SIGBUS 0 #endif /* !NOPAGE_SIGBUS */ #if LINUX_VERSION_CODE > 0x020500 typedef struct page mem_map_t; typedef mem_map_t *vm_nopage_ret_t; #elif LINUX_VERSION_CODE >= 0x020400 typedef mem_map_t* vm_nopage_ret_t; #endif /* LINUX_VERSION_CODE */ static __inline__ vm_nopage_ret_t do_vm_nopage(struct vm_area_struct* vma, unsigned long address) { return 0; /* Disallow mremap */ } #ifdef __AGP__BUILTIN__ #ifdef __ia64__ static __inline__ vm_nopage_ret_t do_vm_cant_nopage(struct vm_area_struct* vma, unsigned long address) { void *dev; unsigned long offset = address - vma->vm_start; unsigned long baddr = VM_OFFSET(vma) + offset; unsigned long mem; struct page *page; // TODO dev = firegl_get_dev_from_vm(vma); // if (firegl_cant_use_agp(dev)) { mem = firegl_get_virt_agp_mem( dev, baddr, vma); if (mem) { page = virt_to_page((unsigned long)__va(mem)); get_page(page); return page; } } return NOPAGE_SIGBUS; /* Disallow mremap */ } #endif /* __ia64__ */ #endif /* __AGP__BUILTIN__ */ static __inline__ vm_nopage_ret_t do_vm_shm_nopage(struct vm_area_struct* vma, unsigned long address) { pgd_t* pgd_p; pmd_t* pmd_p; pte_t pte; unsigned long vma_offset; unsigned long pte_linear; mem_map_t* pMmPage; /* vm_start => start of vm-area, regular address vm_end => end of vm-area, regular address vm_offset/vm_pgoff => start of area, linear address address => requested address, regular address Check range Seems the surrounding framework already does that test - skip it here, anyone does. */ #if 0 if (address < vma->vm_start) return NOPAGE_SIGBUS; /* Address is out of range */ #endif /* Note: vm_end is not member of range but this border hmm, might be used when growing the VMA, not sure - keep it as it is. */ __KE_DEBUG3("start=0x%08lx, " "end=0x%08lx, " "offset=0x%08lx\n", vma->vm_start, vma->vm_end, (unsigned long)__ke_vm_offset(vma)); if (address > vma->vm_end) return NOPAGE_SIGBUS; /* address is out of range */ /* Calculate offset into VMA */ vma_offset = address - vma->vm_start; /* Find the map with the given handle (vm_offset) and get the linear address. */ pte_linear = firegl_get_addr_from_vm(vma); if (!pte_linear) { return NOPAGE_SIGBUS; /* bad address */ } pte_linear += vma_offset; /* Locate responsible kernel PTE for this linear address in paging system of the kernel VM (or is it the init process? - not sure yet) */ PGD_OFFSET_K(pgd_p, pte_linear); PGD_PRESENT(pgd_p); /* locate medium level page table (x86 => nop) */ PMD_OFFSET(pmd_p, pgd_p, pte_linear); PMD_PRESENT(pmd_p); /* locate page table entry itself */ PTE_OFFSET(pte, pmd_p, pte_linear); PTE_PRESENT(pte); /* Pretty straight in new kernel. The pte is represented by the requested pointer to the page table entry. */ pMmPage = pte_page(pte); get_page(pMmPage); /* inc usage count of page */ // __KE_DEBUG3("vm-address 0x%08lx => kernel-page-address 0x%p\n", // address, page_address(pMmPage)); return pMmPage; } /* This routine is intended to remap addresses of a OpenGL context (which is one ore more pages in size) */ static __inline__ vm_nopage_ret_t do_vm_dma_nopage(struct vm_area_struct* vma, unsigned long address) { unsigned long kaddr; mem_map_t* pMmPage; if (address > vma->vm_end) return 0; /* Disallow mremap */ /* Have we ever got an acces from user land into context structure? Resolve the kernel (mem_map/page) address for the VMA-address we got queried about. */ kaddr = firegl_get_addr_from_vm(vma); if (!kaddr) { return NOPAGE_SIGBUS; /* bad address */ } kaddr += (address - vma->vm_start); pMmPage = virt_to_page(kaddr); #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 15) // WARNING WARNINIG WARNNING WARNNING WARNNING WARNNING WARNNING WARNNING // Don't increment page usage count, cause ctx pages are allocated // with drm_alloc_pages, which marks all pages as reserved. Reserved // pages' usage count is not decremented by the kernel during unmap!!! // // For kernel >= 2.6.15, We should reenable this, because the VM sub-system // will decrement the pages' usage count even for the pages marked as reserved // - MC. get_page(pMmPage); /* inc usage count of page */ #endif __KE_DEBUG3("vm-address 0x%08lx => kernel-page-address 0x%p\n", address, page_address(pMmPage)); return pMmPage; } static __inline__ vm_nopage_ret_t do_vm_kmap_nopage(struct vm_area_struct* vma, unsigned long address) { unsigned long kaddr; mem_map_t* pMmPage; if (address > vma->vm_end) return 0; /* Disallow mremap */ if ((pMmPage = (mem_map_t*) firegl_get_pagetable_page_from_vm(vma))) { get_page(pMmPage); return pMmPage; } kaddr = firegl_get_addr_from_vm(vma); if (!kaddr) { return NOPAGE_SIGBUS; /* bad address */ } kaddr += (address - vma->vm_start); __KE_DEBUG3("kaddr=0x%08lx\n", kaddr); pMmPage = virt_to_page(kaddr); __KE_DEBUG3("pMmPage=0x%08lx\n", (unsigned long)pMmPage); get_page(pMmPage); /* inc usage count of page */ __KE_DEBUG3("vm-address 0x%08lx => kernel-page-address 0x%p\n", address, page_address(pMmPage)); return pMmPage; } /** ** ** This routine is intented to locate the page table through the ** pagelist table created earlier in dev-> pcie **/ static __inline__ vm_nopage_ret_t do_vm_pcie_nopage(struct vm_area_struct* vma, unsigned long address) { unsigned long vma_offset; unsigned long i; mem_map_t* pMmPage; struct firegl_pcie_mem* pciemem; unsigned long* pagelist; drm_device_t *dev = (drm_device_t *)firegl_get_dev_from_vm(vma); if (dev == NULL) { __KE_ERROR("dev is NULL\n"); return NOPAGE_SIGBUS; } if (address > vma->vm_end) { __KE_ERROR("address out of range\n"); return NOPAGE_SIGBUS; /* address is out of range */ } pciemem = firegl_get_pciemem_from_addr ( vma, address); if (pciemem == NULL) { __KE_ERROR("No pciemem found! \n"); return NOPAGE_SIGBUS; } pagelist = firegl_get_pagelist_from_vm(vma); if (pagelist == NULL) { __KE_ERROR("No pagelist! \n"); return NOPAGE_SIGBUS; } /** Find offset in vma */ vma_offset = address - vma->vm_start; /** Which entry in the pagelist */ i = vma_offset >> PAGE_SHIFT; pMmPage = virt_to_page(firegl_get_pcie_pageaddr_from_vm(vma,pciemem, i)); get_page(pMmPage); if (page_address(pMmPage) == 0x0) { __KE_ERROR("Invalid page address\n"); return NOPAGE_SIGBUS; } return pMmPage; } static __inline__ vm_nopage_ret_t do_vm_gart_nopage(struct vm_area_struct* vma, unsigned long address) { unsigned long page_addr; unsigned long offset; struct page *page; if (address > vma->vm_end) return NOPAGE_SIGBUS; /* Disallow mremap */ offset = address - vma->vm_start; page_addr = firegl_cmmqs_get_pageaddr_from_vm(vma, offset); page = virt_to_page(page_addr); get_page(page); return page; } #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) static vm_nopage_ret_t vm_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { if (type) *type = VM_FAULT_MINOR; return do_vm_nopage(vma, address); } #ifdef __AGP__BUILTIN__ #ifdef __ia64__ static vm_nopage_ret_t vm_cant_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { if (type) *type = VM_FAULT_MINOR; return do_cant_nopage(vma, address); } #endif /* __ia64__ */ #endif /* __AGP__BUILTIN__ */ /* This function is called when a page of a mmap()'ed area is not currently visible in the specified VMA. Return value is the associated physical address for the requested page. (If not implemented, then the kernel default routine would allocate a new, zeroed page for servicing us) Possible errors: SIGBUS, OutOfMem This routine is intended to remap addresses of SHM SAREA (which is one or more pages in size) */ static vm_nopage_ret_t vm_shm_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { if (type) *type = VM_FAULT_MINOR; return do_vm_shm_nopage(vma, address); } /* This routine is intended to remap addresses of a OpenGL context (which is one ore more pages in size) */ static vm_nopage_ret_t vm_dma_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { if (type) *type = VM_FAULT_MINOR; return do_vm_dma_nopage(vma, address); } static vm_nopage_ret_t vm_kmap_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { if (type) *type = VM_FAULT_MINOR; return do_vm_kmap_nopage(vma, address); } static vm_nopage_ret_t vm_pcie_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { return do_vm_pcie_nopage(vma, address); } static vm_nopage_ret_t vm_gart_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { return do_vm_gart_nopage(vma, address); } #else /* LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,0) */ static vm_nopage_ret_t vm_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_nopage(vma, address); } #ifdef __AGP__BUILTIN__ #ifdef __ia64__ static vm_nopage_ret_t vm_cant_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_cant_nopage(vma, address); } #endif /* __ia64__ */ #endif /* __AGP__BUILTIN__ */ /* This function is called when a page of a mmap()'ed area is not currently visible in the specified VMA. Return value is the associated physical address for the requested page. (If not implemented, then the kernel default routine would allocate a new, zeroed page for servicing us) Possible errors: SIGBUS, OutOfMem This routine is intended to remap addresses of SHM SAREA (which is one or more pages in size) */ static vm_nopage_ret_t vm_shm_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_shm_nopage(vma, address); } /* This routine is intended to remap addresses of a OpenGL context (which is one ore more pages in size) */ static vm_nopage_ret_t vm_dma_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_dma_nopage(vma, address); } static vm_nopage_ret_t vm_kmap_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_kmap_nopage(vma, address); } static vm_nopage_ret_t vm_pcie_nopage(struct vm_area_struct* vma, unsigned long address, int write_access) { return do_vm_pcie_nopage(vma, address); } static vm_nopage_ret_t vm_gart_nopage(struct vm_area_struct* vma, unsigned long address, int *type) { return do_vm_gart_nopage(vma, address); } #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2,6,0) */ void* ATI_API_CALL __ke_vma_file_priv(struct vm_area_struct* vma) { return vma->vm_file->private_data; } unsigned long ATI_API_CALL __ke_vm_start(struct vm_area_struct* vma) { return vma->vm_start; } unsigned long ATI_API_CALL __ke_vm_end(struct vm_area_struct* vma) { return vma->vm_end; } unsigned long ATI_API_CALL __ke_vm_offset(struct vm_area_struct* vma) { #if LINUX_VERSION_CODE < 0x020319 return vma->vm_offset; #else /* LINUX_VERSION_CODE >= 0x020319 */ return vma->vm_pgoff << PAGE_SHIFT; #endif/* LINUX_VERSION_CODE >= 0x020319 */ } char* ATI_API_CALL __ke_vm_flags_str(struct vm_area_struct* vma, char* buf) { *(buf + 0) = vma->vm_flags & VM_READ ? 'r' : '-'; *(buf + 1) = vma->vm_flags & VM_WRITE ? 'w' : '-'; *(buf + 2) = vma->vm_flags & VM_EXEC ? 'x' : '-'; *(buf + 3) = vma->vm_flags & VM_MAYSHARE ? 's' : 'p'; *(buf + 4) = vma->vm_flags & VM_LOCKED ? 'l' : '-'; *(buf + 5) = vma->vm_flags & VM_IO ? 'i' : '-'; *(buf + 6) = 0; return buf; } char* ATI_API_CALL __ke_vm_page_prot_str(struct vm_area_struct* vma, char* buf) { int i = 0; #ifdef __i386__ unsigned int pgprot; pgprot = pgprot_val(vma->vm_page_prot); *(buf + i++) = pgprot & _PAGE_PRESENT ? 'p' : '-'; *(buf + i++) = pgprot & _PAGE_RW ? 'w' : 'r'; *(buf + i++) = pgprot & _PAGE_USER ? 'u' : 's'; *(buf + i++) = pgprot & _PAGE_PWT ? 't' : 'b'; *(buf + i++) = pgprot & _PAGE_PCD ? 'u' : 'c'; *(buf + i++) = pgprot & _PAGE_ACCESSED ? 'a' : '-'; *(buf + i++) = pgprot & _PAGE_DIRTY ? 'd' : '-'; *(buf + i++) = pgprot & _PAGE_PSE ? 'm' : 'k'; *(buf + i++) = pgprot & _PAGE_GLOBAL ? 'g' : 'l'; #endif /* __i386__ */ *(buf + i++) = 0; return buf; } static char *__ke_pte_phys_addr_str(pte_t pte, char *buf, __ke_dma_addr_t* phys_address) { if (pte_present(pte)) { #if defined(__x86_64__) || defined(__ia64__) *phys_address = pte.pte & PAGE_MASK; #else *phys_address = pte_val(pte) & (u64)((u64)PAGE_MASK | (u64)0xf<<32); #endif sprintf(buf, "0x%Lx %c%c%c%c%c%c\n", *phys_address, pte_present (pte) ? 'p' : '-', pte_read (pte) ? 'r' : '-', pte_write (pte) ? 'w' : '-', pte_exec (pte) ? 'x' : '-', pte_dirty (pte) ? 'd' : '-', pte_young (pte) ? 'a' : '-'); } else *buf = 0; return buf; } char* ATI_API_CALL __ke_vm_phys_addr_str(struct vm_area_struct* vma, char* buf, unsigned long virtual_addr, __ke_dma_addr_t* phys_address) { pgd_t* pgd_p; pmd_t* pmd_p; pte_t pte; PGD_OFFSET(vma->vm_mm, pgd_p, virtual_addr); PGD_PRESENT(pgd_p); PMD_OFFSET(pmd_p, pgd_p, virtual_addr); PMD_PRESENT(pmd_p); PTE_OFFSET(pte, pmd_p, virtual_addr); PTE_PRESENT(pte); return __ke_pte_phys_addr_str(pte, buf, phys_address); } void ip_drm_vm_open(struct vm_area_struct* vma) { drm_vm_open(vma); } void ip_drm_vm_close(struct vm_area_struct* vma) { drm_vm_close(vma); } static struct vm_operations_struct vm_ops = { nopage: vm_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; #ifdef __AGP__BUILTIN__ #ifdef __ia64__ static struct vm_operations_struct vm_cant_ops = { nopage: vm_cant_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; #endif /* __ia64_ */ #endif /* __AGP__BUILTIN__ */ static struct vm_operations_struct vm_shm_ops = { nopage: vm_shm_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; static struct vm_operations_struct vm_pci_bq_ops = { nopage: vm_dma_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; static struct vm_operations_struct vm_ctx_ops = { nopage: vm_dma_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; static struct vm_operations_struct vm_pcie_ops = { nopage: vm_pcie_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; static struct vm_operations_struct vm_kmap_ops = { nopage: vm_kmap_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; static struct vm_operations_struct vm_gart_ops = { nopage: vm_gart_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; #ifdef __AGP__BUILTIN__ #ifndef __ia64__ static struct vm_operations_struct vm_agp_bq_ops = { nopage: vm_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; #else static struct vm_operations_struct vm_cant_agp_bq_ops = { nopage: vm_cant_nopage, open: ip_drm_vm_open, close: ip_drm_vm_close, }; #endif /* __ia64_ */ #endif /* __AGP__BUILTIN__ */ int ATI_API_CALL __ke_vm_map(struct file* filp, struct vm_area_struct* vma, unsigned long offset, enum __ke_vm_maptype type, int readonly) { unsigned int pages; __KE_DEBUG3("start=0x%08lx, " "end=0x%08lx, " "offset=0x%08lx\n", vma->vm_start, vma->vm_end, (unsigned long)offset); switch (type) { case __KE_ADPT: { #ifdef __ia64__ struct page *page = virt_to_page((unsigned long)__va(VM_OFFSET(vma))); if (!VALID_PAGE(page) || PageReserved(page)) #else if (offset >= __pa(high_memory)) #endif { #ifdef __i386__ if (boot_cpu_data.x86 > 3) { pgprot_val(vma->vm_page_prot) |= _PAGE_PCD; pgprot_val(vma->vm_page_prot) &= ~_PAGE_PWT; } #endif /* __i386__ */ #ifdef __ia64__ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); #endif /* __ia64__ */ vma->vm_flags |= VM_IO; /* not in core dump */ } if (REMAP_PAGE_RANGE(vma,offset)) { __KE_DEBUG(REMAP_PAGE_RANGE_STR " failed\n"); return -EAGAIN; } vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_ops = &vm_ops; } break; case __KE_SHM: vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_ops = &vm_shm_ops; break; case __KE_SG: pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; #if LINUX_VERSION_CODE <= 0x02040e /* KERNEL_VERSION(2,4,14) */ vma->vm_flags |= VM_LOCKED | VM_SHM; /* Don't swap */ #else vma->vm_flags |= VM_RESERVED; #endif //vma->vm_flags |= VM_SHM | VM_LOCKED; /* DDDDDDDDDDon't swap */ //vma->vm_mm->locked_vm += pages; /* Kernel tracks aqmount of locked pages */ vma->vm_ops = &vm_pcie_ops; break; case __KE_CTX: pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; vma->vm_flags |= VM_LOCKED | VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_mm->locked_vm += pages; /* Kernel tracks aqmount of locked pages */ vma->vm_ops = &vm_ctx_ops; break; case __KE_PCI_BQS: pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; vma->vm_flags |= VM_LOCKED | VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_mm->locked_vm += pages; /* Kernel tracks aqmount of locked pages */ vma->vm_ops = &vm_pci_bq_ops; break; #ifdef __AGP__BUILTIN__ case __KE_AGP: // if(dev->agp->cant_use_aperture == 1) #ifdef __ia64__ { /* * On some systems we can't talk to bus dma address from * the CPU, so for memory of type DRM_AGP, we'll deal * with sorting out the real physical pages and mappings * in our page fault handler */ vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_ops = &vm_cant_ops; } #else // else { if (offset >= __pa(high_memory)) vma->vm_flags |= VM_IO; /* not in core dump */ if (REMAP_PAGE_RANGE(vma,offset)) { __KE_DEBUG(REMAP_PAGE_RANGE_STR " failed\n"); return -EAGAIN; } #ifdef __x86_64__ vma->vm_flags |= VM_RESERVED; #else vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ #endif vma->vm_ops = &vm_ops; } #endif break; case __KE_AGP_BQS: // if(dev->agp->cant_use_aperture == 1) #ifdef __ia64__ { /* * On some systems we can't talk to bus dma address from * the CPU, so for memory of type DRM_AGP, we'll deal * with sorting out the real physical pages and mappings * in our page fault handler */ vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ vma->vm_ops = &vm_cant_agp_bq_ops; } // else #else { if (offset >= __pa(high_memory)) vma->vm_flags |= VM_IO; /* not in core dump */ if (REMAP_PAGE_RANGE(vma,offset)) { __KE_DEBUG(REMAP_PAGE_RANGE_STR " failed\n"); return -EAGAIN; } #ifdef __x86_64__ vma->vm_flags |= VM_RESERVED; #else vma->vm_flags |= VM_SHM | VM_RESERVED; /* Don't swap */ #endif vma->vm_ops = &vm_agp_bq_ops; } #endif break; #endif /* __AGP__BUILTIN__ */ case __KE_KMAP: vma->vm_flags |= VM_SHM | VM_RESERVED; vma->vm_ops = &vm_kmap_ops; if (readonly && (vma->vm_flags & VM_WRITE)) { __KE_DEBUG("ERROR: cannot map a readonly map with PROT_WRITE!\n"); return -EINVAL; // write not allowed - explicitly fail the map! } break; case __KE_GART_USWC: case __KE_GART_CACHEABLE: #if LINUX_VERSION_CODE <= 0x02040e /* KERNEL_VERSION(2,4,14) */ vma->vm_flags |= VM_LOCKED | VM_SHM; /* Don't swap */ #else vma->vm_flags |= VM_RESERVED; #endif vma->vm_ops = &vm_gart_ops; break; default: /* This should never happen anyway! */ __KE_ERROR("__ke_vm_map: Unknown type %d\n", type); return -EINVAL; } if (readonly) { vma->vm_flags &= ~(VM_WRITE | VM_MAYWRITE); #if defined(__i386__) pgprot_val(vma->vm_page_prot) &= ~_PAGE_RW; #else vma->vm_page_prot = __pgprot(pte_val(pte_wrprotect( __pte(pgprot_val(vma->vm_page_prot))))); #endif } vma->vm_file = filp; /* Needed for drm_vm_open() */ return 0; } int __ke_agp_try_unsupported = 0; #define _X(_x) __fgl_##_x extern int _X(agp_init)(void); extern void _X(agp_cleanup)(void); extern int _X(agp_try_unsupported); extern struct _agp_memory *_X(agp_allocate_memory_phys_list)(size_t, u32, u64 *); #define FIREGL_agp_init _X(agp_init) #define FIREGL_agp_cleanup _X(agp_cleanup) #define FIREGL_agp_free_memory _X(agp_free_memory) #define FIREGL_agp_allocate_memory _X(agp_allocate_memory) #ifdef FGL #define FIREGL_agp_allocate_memory_phys_list _X(agp_allocate_memory_phys_list) #endif #define FIREGL_agp_bind_memory _X(agp_bind_memory) #define FIREGL_agp_unbind_memory _X(agp_unbind_memory) #define FIREGL_agp_enable _X(agp_enable) #define FIREGL_agp_backend_acquire _X(agp_backend_acquire) #define FIREGL_agp_backend_release _X(agp_backend_release) #define FIREGL_agp_memory _X(agp_memory) unsigned int __ke_firegl_agpgart_inuse = AGPGART_INUSE_NONE; #if defined(CONFIG_AGP) || defined(CONFIG_AGP_MODULE) /*****************************************************************************/ #if LINUX_VERSION_CODE >= 0x02060b typedef struct { void (*free_memory)(struct agp_memory *); struct agp_memory * (*allocate_memory)(size_t, u32); int (*bind_memory)(struct agp_memory *, off_t); int (*unbind_memory)(struct agp_memory *); void (*enable)(u32); int (*acquire)(void); void (*release)(void); int (*copy_info)(struct agp_kern_info *); } drm_agp_t; #if LINUX_VERSION_CODE >= 0x02060c /* In Linux >= 2.6.12, due to support for multiple AGP bridges, some * AGP functions need a pointer to the AGP bridge. __ke_agp_acquire * stores a pointer to the pci device in a global * variable. firegl_wrap_agp_backend_acquire below uses it to retrieve * the pointer to the bridge and stores it in the global variable * firegl_agp_bridge. All AGP functions that need the bridge pointer * are wrapped here and get the global bridge pointer. */ static struct agp_bridge_data *firegl_agp_bridge = NULL; static struct pci_dev *firegl_pci_device = NULL; static struct agp_memory * firegl_wrap_agp_allocate_memory (size_t pg_count, u32 type) { return agp_allocate_memory (firegl_agp_bridge, pg_count, type); } static void firegl_wrap_agp_enable (u32 mode) { agp_enable (firegl_agp_bridge, mode); } static int firegl_wrap_agp_backend_acquire (void) { firegl_agp_bridge = agp_backend_acquire (firegl_pci_device); return firegl_agp_bridge != NULL ? 0 : -EBUSY; } static void firegl_wrap_agp_backend_release (void) { agp_backend_release (firegl_agp_bridge); firegl_agp_bridge = NULL; } static int firegl_wrap_agp_copy_info (struct agp_kern_info *kinfo) { return agp_copy_info (firegl_agp_bridge, kinfo); } static const drm_agp_t drm_agp = { &agp_free_memory, &firegl_wrap_agp_allocate_memory, &agp_bind_memory, &agp_unbind_memory, &firegl_wrap_agp_enable, &firegl_wrap_agp_backend_acquire, &firegl_wrap_agp_backend_release, &firegl_wrap_agp_copy_info }; #else static const drm_agp_t drm_agp = { &agp_free_memory, &agp_allocate_memory, &agp_bind_memory, &agp_unbind_memory, &agp_enable, &agp_backend_acquire, &agp_backend_release, &agp_copy_info }; #endif /* >= 2.6.12 */ #else #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) static const drm_agp_t drm_agp = { &agp_free_memory, &agp_allocate_memory, &agp_bind_memory, &agp_unbind_memory, &agp_enable, &agp_backend_acquire, &agp_backend_release, &agp_copy_info }; #else extern drm_agp_t drm_agp; #endif #endif static const drm_agp_t *drm_agp_module_stub = NULL; #define AGP_FUNCTIONS 8 #define AGP_AVAILABLE(func) (drm_agp_module_stub && drm_agp_module_stub-> func ) #define AGP_FUNC(func) (*drm_agp_module_stub-> func ) // note: avoid ##-tokens with latest GCC (i.e. RedHat 7.1 beta) #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,0) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) #define DRM_AGP_MODULE_GET (drm_agp_t *)inter_module_get_request("drm_agp","drm_agp") #define DRM_AGP_MODULE_PUT inter_module_put("drm_agp") static int ATI_API_CALL __ke_firegl_agpgart_available(void) { int retval; if (__ke_agp_try_unsupported) _X(agp_try_unsupported) = __ke_agp_try_unsupported; retval = FIREGL_agp_init(); if (retval) { __KE_INFO("Initialization of built-in AGP-support failed (ret=%d).\n", retval); __ke_firegl_agpgart_inuse = AGPGART_INUSE_NONE; return 0; } __ke_firegl_agpgart_inuse = INTERNAL_AGPGART_INUSE; drm_agp_module_stub = &drm_agp; __KE_INFO("Initialization of built-in AGP-support successful.\n"); return 1; } static int ATI_API_CALL __ke_agpgart_available(__ke_pci_dev_t *pcidev, int use_internal) { unsigned int found = 0; drm_agp_module_stub = DRM_AGP_MODULE_GET; if (!drm_agp_module_stub) { __KE_DEBUG("getting module stub failed for AGP/GART kernel module\n"); return 0; /* failed */ } __ke_firegl_agpgart_inuse = KERNEL24_AGPGART_INUSE; if (drm_agp_module_stub->free_memory) { __KE_DEBUG("agp_free_memory resolves to 0x%08lx\n", drm_agp_module_stub->free_memory); found++; } if (drm_agp_module_stub->allocate_memory) { __KE_DEBUG("agp_allocate_memory resolves to 0x%08lx\n", drm_agp_module_stub->allocate_memory); found++; } if (drm_agp_module_stub->bind_memory) { __KE_DEBUG("agp_bind_memory resolves to 0x%08lx\n", drm_agp_module_stub->bind_memory); found++; } if (drm_agp_module_stub->unbind_memory) { __KE_DEBUG("agp_unbind_memory resolves to 0x%08lx\n", drm_agp_module_stub->unbind_memory); found++; } if (drm_agp_module_stub->enable) { __KE_DEBUG("agp_enable resolves to 0x%08lx\n", drm_agp_module_stub->enable); found++; } if (drm_agp_module_stub->acquire) { __KE_DEBUG("agp_acquire resolves to 0x%08lx\n", drm_agp_module_stub->acquire); found++; } if (drm_agp_module_stub->release) { __KE_DEBUG("agp_release resolves to 0x%08lx\n", drm_agp_module_stub->release); found++; } if (drm_agp_module_stub->copy_info) { __KE_DEBUG("agp_copy_info resolves to 0x%08lx\n", drm_agp_module_stub->copy_info); found++; } if (found == AGP_FUNCTIONS) { if (AGP_FUNC(acquire)() == 0) { AGP_FUNC(release)(); return 1; /* success */ } } if (found == 0) __KE_DEBUG("AGP/GART kernel module not found\n"); else { __KE_DEBUG("AGP/GART kernel module present but API is incomplete\n"); /* do a complete cleanup of our entry point table now */ __ke_agp_uninit(); } return 0; /* failed */ } #else // !LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,0) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) static int ATI_API_CALL __ke_agpgart_available(__ke_pci_dev_t *pcidev, int use_internal) { drm_agp_module_stub = &drm_agp; __ke_firegl_agpgart_inuse = KERNEL26_AGPGART_INUSE; { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) firegl_pci_device = (struct pci_dev*)(void*)pcidev; #endif if (AGP_FUNC(acquire)() == 0) { AGP_FUNC(release)(); return 1; /* success */ } } { __KE_DEBUG("AGP/GART kernel module present but API is incomplete\n"); /* do a complete cleanup of our entry point table now */ __ke_agp_uninit(); } return 0; /* failed */ } static int ATI_API_CALL __ke_firegl_agpgart_available(void) { __KE_INFO("Internal AGP is not supported in 2.6 kernel.\n"); return 0; } #endif // LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,0) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) int ATI_API_CALL __ke_agp_available(__ke_pci_dev_t *pcidev, int use_internal) { int available = __ke_agpgart_available(pcidev, use_internal); if ( available ) { if ( use_internal ) { __KE_INFO("Internal AGP support requested, but kernel AGP support active.\n"); __KE_INFO("Have to use kernel AGP support to avoid conflicts.\n"); } if ( __ke_moduleflags & __KE_FLAG_DISABLE_AGPGART ) { __KE_ERROR("Have to use kernel AGP support, but module parameters forbid that\n"); __ke_agp_uninit(); available = 0; } } else { available = __ke_firegl_agpgart_available(); } return available; } void ATI_API_CALL __ke_agp_uninit(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,0) && LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) if( (__ke_firegl_agpgart_inuse & INTERNAL_AGPGART_INUSE) ) { FIREGL_agp_cleanup(); } else if( (__ke_firegl_agpgart_inuse & KERNEL24_AGPGART_INUSE) && drm_agp_module_stub ) { DRM_AGP_MODULE_PUT; } #endif __ke_firegl_agpgart_inuse = AGPGART_INUSE_NONE; drm_agp_module_stub = NULL; } void ATI_API_CALL __ke_agp_free_memory(struct _agp_memory* handle) { if (AGP_AVAILABLE(free_memory)) AGP_FUNC(free_memory)(handle); } struct _agp_memory* ATI_API_CALL __ke_agp_allocate_memory(__ke_size_t pages, unsigned long type) { if (AGP_AVAILABLE(allocate_memory)) return AGP_FUNC(allocate_memory)(pages, type); else return NULL; } int ATI_API_CALL __ke_agp_bind_memory(struct _agp_memory* handle, __ke_off_t start) { if (AGP_AVAILABLE(bind_memory)) return AGP_FUNC(bind_memory)(handle, start); else return -EINVAL; } int ATI_API_CALL __ke_agp_unbind_memory(struct _agp_memory* handle) { if (AGP_AVAILABLE(unbind_memory)) return AGP_FUNC(unbind_memory)(handle); else return -EINVAL; } int ATI_API_CALL __ke_agp_enable(unsigned long mode) { if (AGP_AVAILABLE(enable)) { AGP_FUNC(enable)(mode); return 0; } else return -EINVAL; } int ATI_API_CALL __ke_read_agp_caps_registers(__ke_pci_dev_t* dev, unsigned int *caps) { u8 capndx; u32 cap_id; if (!caps) return -EINVAL; if (!(struct pci_dev*)(void *)dev) return -ENODEV; pci_read_config_byte((struct pci_dev*)(void *)dev, 0x34, &capndx); if (capndx == 0x00) return -ENODATA; do { // search capability list for AGP caps pci_read_config_dword((struct pci_dev*)(void *)dev, capndx, &cap_id); if ((cap_id & 0xff) == 0x02) { pci_read_config_dword((struct pci_dev*)(void *)dev, capndx + 0, &(caps[0])); /* AGP CAPPTR */ pci_read_config_dword((struct pci_dev*)(void *)dev, capndx + 4, &(caps[1])); /* AGP STATUS */ pci_read_config_dword((struct pci_dev*)(void *)dev, capndx + 8, &(caps[2])); /* AGP COMMAND */ return 0; /* success */ } capndx = (cap_id >> 8) & 0xff; } while (capndx != 0x00); return -ENODATA; } int ATI_API_CALL __ke_agp_acquire(__ke_pci_dev_t* dev) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) firegl_pci_device = (struct pci_dev*)(void*)dev; #endif if (AGP_AVAILABLE(acquire)) return AGP_FUNC(acquire)(); else return -EINVAL; } void ATI_API_CALL __ke_agp_release(void) { if (AGP_AVAILABLE(release)) AGP_FUNC(release)(); } void ATI_API_CALL __ke_agp_copy_info(__ke_agp_kern_info_t* info) { struct pci_dev *device = NULL; memset(info, 0, sizeof(__ke_agp_kern_info_t)); if (AGP_AVAILABLE(copy_info)) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,71) struct #endif agp_kern_info kern; #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,12) if (firegl_agp_bridge == NULL) { AGP_FUNC(acquire)(); AGP_FUNC(copy_info)(&kern); AGP_FUNC(release)(); } else #endif AGP_FUNC(copy_info)(&kern); device = kern.device; info->version.major = kern.version.major; info->version.minor = kern.version.minor; if( kern.device ) { info->vendor = kern.device->vendor; info->device = kern.device->device; } info->mode = kern.mode; info->aper_base = kern.aper_base; info->aper_size = kern.aper_size; info->max_memory = kern.max_memory; info->current_memory = kern.current_memory; #if LINUX_VERSION_CODE <= 0x020408 info->cant_use_aperture = 0; info->page_mask = ~(0xfff); #else info->cant_use_aperture = kern.cant_use_aperture; info->page_mask = kern.page_mask; #endif } /* FGL_FIX: some chipset drivers do not read the mode member from hardware */ if( device ) { if( !info->mode ) { u8 capptr; capptr = pci_find_capability(device, PCI_CAP_ID_AGP); if( capptr ) { u32 tmp; pci_read_config_dword(device, capptr + PCI_AGP_STATUS, &tmp); info->mode = tmp; /* note: unsigned int (32/64) = u32; */ } } } } unsigned long ATI_API_CALL __ke_agp_memory_handle(struct _agp_memory* handle) { return (unsigned long)handle->memory; } unsigned long ATI_API_CALL __ke_agp_memory_page_count(struct _agp_memory* handle) { return handle->page_count; } #else //!defined(CONFIG_AGP) || defined(CONFIG_AGP_MODULE) int ATI_API_CALL __ke_agp_available(__ke_pci_dev_t *pcidev, int use_internal) { return 0; } void ATI_API_CALL __ke_agp_uninit( void ) { } void ATI_API_CALL __ke_agp_free_memory(struct _agp_memory* handle) { } struct _agp_memory* ATI_API_CALL __ke_agp_allocate_memory(__ke_size_t pages, unsigned long type) { return NULL; } int ATI_API_CALL __ke_agp_bind_memory(struct _agp_memory* handle, __ke_off_t start) { return -EINVAL; } int ATI_API_CALL __ke_agp_unbind_memory(struct _agp_memory* handle) { return -EINVAL; } int ATI_API_CALL __ke_agp_enable(unsigned long mode) { return -EINVAL; } int ATI_API_CALL __ke_read_agp_caps_registers(__ke_pci_dev_t* dev, unsigned int *caps) { return -EINVAL; } int ATI_API_CALL __ke_agp_acquire(__ke_pci_dev_t* dev) { return -EINVAL; } void ATI_API_CALL __ke_agp_release(void) { } int ATI_API_CALL __ke_agp_read_caps_registers(__ke_pci_dev_t* dev, unsigned int *caps) { return -EINVAL; } void ATI_API_CALL __ke_agp_copy_info(__ke_agp_kern_info_t* info) { } unsigned long ATI_API_CALL __ke_agp_memory_handle(struct _agp_memory* handle) { return 0; } unsigned long ATI_API_CALL __ke_agp_memory_page_count(struct _agp_memory* handle) { return 0; } #endif //defined(CONFIG_AGP) || defined(CONFIG_AGP_MODULE) #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) struct _agp_memory* ATI_API_CALL __ke_agp_allocate_memory_phys_list( __ke_size_t pages, unsigned long type, __ke_dma_addr_t *phys_addr) { return NULL; } #else // LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) #ifdef FGL struct _agp_memory * ATI_API_CALL __ke_agp_allocate_memory_phys_list(__ke_size_t pages, unsigned long type, __ke_dma_addr_t * phys_addr) { return (struct _agp_memory *)FIREGL_agp_allocate_memory_phys_list(pages, type, phys_addr); } #endif #endif // LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) int ATI_API_CALL __ke_smp_processor_id(void) { return (int)(smp_processor_id()); } void ATI_API_CALL __ke_smp_call_function( void (*ATI_API_CALL func)(void *info) ) { #ifdef CONFIG_SMP smp_call_function( func, NULL, 0, 1 ); #endif } #if !defined(__x86_64__) && !defined(__ia64__) #define vendorStringLen 12 static inline int check_cpu_vendor(const char* vendorName) { int sVendor[vendorStringLen/sizeof(int)]; __asm ( "\txor %%eax, %%eax\n" // function #0: get vendor string "\tcpuid\n" // perform CPUID "\tmov %%ebx, %0\n" // store 3 times 4 bytes of vendor string "\tmov %%edx, %1\n" "\tmov %%ecx, %2\n" : "=m" (sVendor[0]), "=m" (sVendor[1]), "=m"(sVendor[2]) // results go to this location : // no input : "eax", "ebx", "ecx", "edx" // modifiys this registers ); if( strncmp((char*)sVendor,vendorName,vendorStringLen)==0 ) return 1; // thats a match else return 0; // does not match } int ATI_API_CALL __ke_is_athlon(void) { register int bAthlon; __asm ( // Check for CPUID instruction "\tpushf\n" //; save EFLAGS "\tpop %%eax\n" //; store EFLAGS in EAX "\tmov %%eax, %%ebx\n" //; save in EBX for later testing "\txor $0x00200000, %%eax\n" //; toggle bit 21 "\tpush %%eax\n" //; put to stack "\tpopf\n" //; save changed EAX to EFLAGS "\tpushf\n" //; push EFLAGS to TOS "\tpop %%eax\n" //; store EFLAGS in EAX "\tcmp %%ebx, %%eax\n" //; see if bit 21 has changed "\tjz NO_ATHLON\n" //; if no change, no CPUID // Check for extended functions "\tmov $0x80000000, %%eax\n" //; query for extended functions "\tCPUID\n" //; get extended function limit "\tcmp $0x80000000, %%eax\n" //; is 8000_0001h supported? "\tjbe NO_ATHLON\n" //; if not, 3DNow! command set not supported "\tmov $0x80000001, %%eax\n" //; setup extended function 1 "\tCPUID\n" //; call the function "\ttest $0x80000000, %%edx\n" //; test bit 31 "\tjz NO_ATHLON\n" //; 3DNow! command set not supported "\tmov $1, %%eax\n" "\tjmp DONE\n" "NO_ATHLON:\n" "\txor %%eax, %%eax\n" "DONE:\n" "\tmov %%eax, %0\n" //; store to bAthlon // optimizer hints : "=r" (bAthlon) // let compiler choose output register : // no input selection for compiler : "eax", "ebx", "ecx", "edx" // modifiys ); if( bAthlon ) { if( !check_cpu_vendor("AuthenticAMD") ) bAthlon = 0; } return bAthlon; } /*======================================================================= * amd_adv_spec_cache_feature * * Test if this is an AMD Athlon processor that exposes a conflicting * cache attribute bug in the kernel. =======================================================================*/ #if ( (PAGE_ATTR_FIX == 1) || (LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,19)) ) // not used #else /* Standard macro to see if a specific flag is changeable */ static inline int flag_is_changeable_p(u32 flag) { u32 f1, f2; asm("pushfl\n\t" "pushfl\n\t" "popl %0\n\t" "movl %0,%1\n\t" "xorl %2,%0\n\t" "pushl %0\n\t" "popfl\n\t" "pushfl\n\t" "popl %0\n\t" "popfl\n\t" : "=&r" (f1), "=&r" (f2) : "ir" (flag)); return ((f1^f2) & flag) != 0; } /* Probe for the CPUID instruction */ static int __init have_cpuid_p(void) { return flag_is_changeable_p(X86_EFLAGS_ID); } #endif int ATI_API_CALL __ke_amd_adv_spec_cache_feature(void) { #if ( (PAGE_ATTR_FIX == 1) || (LINUX_VERSION_CODE >= KERNEL_VERSION(2,4,19)) ) /* the kernel already does provide a fix for the AMD Athlon big page attribute / cache flush data consistency system bug on its own. (AMD claimed that CPU cache behaviour for such pages is not specified.) - kernel 2.4.19 has the early fix which just removes some page attributes. - later kernels do have implementeed the full big page split code fix. */ #else /* PAGE_ATTR_FIX */ /* the kernel does not provide any fix for the AMD problem. */ char vendor_id[16]; int ident; int family, model; /* Must have CPUID */ if(!have_cpuid_p()) goto donthave; if(cpuid_eax(0)<1) goto donthave; /* Must be x86 architecture */ cpuid(0, &ident, (int *)&vendor_id[0], (int *)&vendor_id[8], (int *)&vendor_id[4]); if (memcmp(vendor_id, "AuthenticAMD", 12)) goto donthave; ident = cpuid_eax(1); family = (ident >> 8) & 0xf; model = (ident >> 4) & 0xf; if (((family == 6) && (model >= 6)) || (family == 15)) { return 1; } donthave: #endif /* PAGE_ATTR_FIX */ return 0; } int ATI_API_CALL __ke_has_PSE(void) { #if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,71) if (test_bit(X86_FEATURE_PSE, &boot_cpu_data.x86_capability)) #else if (cpu_has_pse) #endif { return 1; } return 0; } #endif /* !__x86_64__ */ /** \brief Kernel Abstraction Services (KAS) * * TODO: detailed comments, move to a separate file(s), license * */ /** \brief Naming convention * * Externally visible interfaces prefixed with 'KAS_' * Internal helpers prefixed with 'kas' * */ /** \brief KAS context type definition */ typedef struct tag_kasContext_t { unsigned long exec_level_invalid; /* Used if execution level is unknown */ unsigned long exec_level_regular; /* Execution level of regular thread */ unsigned long exec_level_idh; /* Execution level of interrupt handler */ unsigned long exec_level_ih; /* Execution level of interrupt deferred handler */ spinlock_t lock_idh; /* Spinlock for interrupt deferred handler */ spinlock_t lock_ih; /* Spinlock for interrupt handler */ } kasContext_t; /** \brief KAS context */ static kasContext_t kasContext; /** \brief Kernel support required to enable KAS */ #if defined(cmpxchg) && \ defined(xchg) && \ !defined(CONFIG_M386) #define KAS_ATOMIC_OPERATIONS_SUPPORT #endif /** \brief Check whether current kernel fits KAS requirements and restrictions * * \return Nonzero on success, zero on fail * */ static int kasCheckKernelSupport(void) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT /* We use cmpxchg and xchg for atomic exchange operations with pointers. * Since Linux implementation casts parameters to unsigned long, here we * are making sure casting will be safe */ return (sizeof(void*) == sizeof(unsigned long) ? 1 : 0); #else return 0; #endif } /** \brief Storage for execution level(s) */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) /* SMP support for 2.6.0 and higher */ DEFINE_PER_CPU(unsigned long, kasExecutionLevel); #define KAS_EXECUTION_LEVEL_SUPPORT 1 #else #ifndef __SMP__ /* Only uni-CPU support for lower than 2.6.0 */ static unsigned long kasExecutionLevel; #define __get_cpu_var(v) v /* To use the same code for both 2.6 and lower non-SMP */ #define KAS_EXECUTION_LEVEL_SUPPORT 1 #else /* Execution levels are not currently supported for * SMP platforms running kernel less than 2.6 */ #define KAS_EXECUTION_LEVEL_SUPPORT 0 #endif #endif /** \brief Initialize support for execution levels * * This function must be called before interrupt system is initialized. * * \param level_init Value to init execution level(s) * * \return Nonzero on success, zero on fail * */ static int kasInitExecutionLevels(unsigned long level_init) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) unsigned int p; for_each_cpu_mask(p, cpu_possible_map) { preempt_disable(); per_cpu(kasExecutionLevel, p) = level_init; preempt_enable(); } #else #ifndef __SMP__ kasExecutionLevel = level_init; #endif #endif return KAS_EXECUTION_LEVEL_SUPPORT; } /** \brief Initialize KAS * * \param pinit Pointer to KAS initalization structure * * \return Nonzero on success, 0 on fail * */ unsigned int KAS_Initialize(KAS_Initialize_t* pinit) { if (!kasCheckKernelSupport()) { return 0; } spin_lock_init(&kasContext.lock_idh); spin_lock_init(&kasContext.lock_ih); kasContext.exec_level_invalid = pinit->exec_level_invalid; kasContext.exec_level_regular = pinit->exec_level_regular; kasContext.exec_level_idh = pinit->exec_level_idh; kasContext.exec_level_ih = pinit->exec_level_ih; return kasInitExecutionLevels(pinit->exec_level_init); } /** \brief Set execution level for the current processor * * This function is permitted to be called only from * an interrupt handler or from a tasklet, since these two types of callbacks * are guaranteed not to be rescheduled to another CPU. We need to consider * this condition because we use per-CPU variables in this function without * disabling of preemption (to minimize performance losses) * * \param level Level to set * * \return Previous value of the execution level for the current processor * */ static unsigned long kasSetExecutionLevel(unsigned long level) { unsigned long orig_level; #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) || !defined(__SMP__) orig_level = __get_cpu_var(kasExecutionLevel); __get_cpu_var(kasExecutionLevel) = level; #else /* It is not supposed to reach this point because of fail of KAS initialization. * However, we need to make the compiler happy about the return value */ orig_level = kasContext.exec_level_invalid; #endif return orig_level; } /** \brief Internal helper to get execution level for the current processor * * \return Execution level for the current processor * */ unsigned long kas_GetExecutionLevel(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) || !defined(__SMP__) return __get_cpu_var(kasExecutionLevel); #else /* It is not supposed to reach this point because of fail of KAS initialization. * However, we need to make the compiler happy about the return value */ return kasContext.exec_level_invalid; #endif } /** \brief Type definition for kas_spin_lock() parameter */ typedef struct tag_kas_spin_lock_info_t { unsigned int routine_type; /* [IN] Routine type spinlock might be acquired from */ spinlock_t* plock; /* [IN] Pointer to an OS spinlock object */ unsigned int acquire_type; /* [OUT] Type of acquired spinlock */ unsigned long flags; /* [OUT] CPU flags */ } kas_spin_lock_info_t; /** \brief Type definition for kas_spin_unlock() parameter */ typedef struct tag_kas_spin_unlock_info_t { spinlock_t* plock; /* [IN] Pointer to an OS spinlock object */ unsigned int acquire_type; /* [IN] Type of the spinlock */ unsigned long flags; /* [IN] CPU flags */ } kas_spin_unlock_info_t; /** \brief Internal helper to acquire a spin lock depending on the routine type and current execution level * * \param lock_info Pointer to the spinlock parameter/returned data structure * * \return Nonzero on success, zero on fail * */ static unsigned int kas_spin_lock(kas_spin_lock_info_t* lock_info) { unsigned long flags; unsigned int ret = 0; unsigned long exec_level = kas_GetExecutionLevel(); lock_info->acquire_type = KAS_SPINLOCK_TYPE_INVALID; lock_info->flags = 0; switch (lock_info->routine_type) { case KAS_ROUTINE_TYPE_REGULAR: if (exec_level == kasContext.exec_level_regular) { spin_lock(lock_info->plock); lock_info->acquire_type = KAS_SPINLOCK_TYPE_REGULAR; } break; case KAS_ROUTINE_TYPE_IDH: if (exec_level == kasContext.exec_level_regular) { spin_lock_bh(lock_info->plock); lock_info->acquire_type = KAS_SPINLOCK_TYPE_IDH; } else if (exec_level == kasContext.exec_level_idh) { spin_lock(lock_info->plock); lock_info->acquire_type = KAS_SPINLOCK_TYPE_REGULAR; } break; case KAS_ROUTINE_TYPE_IH: if (exec_level == kasContext.exec_level_regular || exec_level == kasContext.exec_level_idh) { spin_lock_irqsave(lock_info->plock, flags); lock_info->acquire_type = KAS_SPINLOCK_TYPE_IH; lock_info->flags = flags; } else if (exec_level == kasContext.exec_level_ih) { spin_lock(lock_info->plock); lock_info->acquire_type = KAS_SPINLOCK_TYPE_REGULAR; } break; default: break; } if (lock_info->acquire_type != KAS_SPINLOCK_TYPE_INVALID) { ret = 1; } return ret; } /** \brief Internal helper to release a spin lock acquired with kas_spin_lock() * * \param unlock_info Pointer to the parameter data structure * * \return Nonzero on success, zero on fail * */ static unsigned int kas_spin_unlock(kas_spin_unlock_info_t* unlock_info) { unsigned long flags; unsigned ret = 1; switch (unlock_info->acquire_type) { case KAS_SPINLOCK_TYPE_REGULAR: spin_unlock(unlock_info->plock); break; case KAS_SPINLOCK_TYPE_IDH: spin_unlock_bh(unlock_info->plock); break; case KAS_SPINLOCK_TYPE_IH: flags = unlock_info->flags; spin_unlock_irqrestore(unlock_info->plock, flags); break; default: ret = 0; break; } return ret; } /** \brief External interface to get execution level for the current processor * * \return Execution level for the current processor * */ unsigned long KAS_GetExecutionLevel(void) { return kas_GetExecutionLevel(); } /** \brief Execute Interrupt Handling Routine * * This service is supposed to be called only during interrupt handling * (the device interrupt must be disabled when calling this service) * Interrupt Handling Routine must fit all requirements to interrupt handlers * * \param ih_routine Routine to run * \param if_context Pointer to context to pass to the routine * * \return Value returned by the ih_routine * */ unsigned int KAS_Ih_Execute(KAS_IhRoutine_t ih_routine, void* ih_context) { unsigned int ret; unsigned long orig_level; spin_lock(&kasContext.lock_ih); orig_level = kasSetExecutionLevel(kasContext.exec_level_ih); ret = ih_routine(ih_context); kasSetExecutionLevel(orig_level); spin_unlock(&kasContext.lock_ih); return ret; } /** \brief Type definition for Interrupt Deferred Handler (IDH) helper routine * * The helper is required to deal set the required execution level for the * time while the routine is being executed * */ typedef void (*kasIdhRoutine_t)(void* pIdhContext); /** \brief Type definition of the structure describing IDH object */ typedef struct tag_kasIdh_t { struct tasklet_struct tasklet; kasIdhRoutine_t routine; void* context; } kasIdh_t; /** \brief IDH helper routine * * This function will called by the OS with the following conditions valid: * - interrupts are enabled * - this function may be interrupted by the interrupt handler for the same device * - this function won't be interrupted by regular threads * - the rest of requirements for interrupt handlers applies * * \param context pointer to the routine context * * \return None * */ static void kasIdhRoutineHelper(unsigned long context) { unsigned long orig_level; kasIdh_t* idh_obj = (kasIdh_t*)context; spin_lock(&kasContext.lock_idh); orig_level = kasSetExecutionLevel(kasContext.exec_level_idh); idh_obj->routine(idh_obj->context); kasSetExecutionLevel(orig_level); spin_unlock(&kasContext.lock_idh); } /** \brief Return IDH object size * * \return IDH object size in bytes * */ unsigned int KAS_Idh_GetObjectSize() { return sizeof(kasIdh_t); } /** \brief Initialize IDH object * * \param hIdh handle of (pointer to) the IDH object * \param pfnIdhRoutine pointer to the IDH routine * \param pIdhContext context pointer to be passed to the IDH routine * * \return Nonzero (always success) * */ unsigned int KAS_Idh_Initialize(void* hIdh, void* pfnIdhRoutine, void* pIdhContext) { kasIdh_t* idh_obj = (kasIdh_t*)hIdh; idh_obj->routine = (kasIdhRoutine_t)pfnIdhRoutine; idh_obj->context = pIdhContext; tasklet_init(&(idh_obj->tasklet), kasIdhRoutineHelper, (unsigned long) idh_obj); return 1; } /** \brief Queue IDH * * \param hIdh handle of (pointer to) the IDH object * * \return Nonzero (always success) * */ unsigned int KAS_Idh_Queue(void* hIdh) { kasIdh_t* idh_obj = (kasIdh_t*)hIdh; tasklet_schedule(&(idh_obj->tasklet)); return 1; } /** \brief Type definition for a routine to execute with level synchonization */ typedef void (*kasSyncRoutine_t)(void* pContext); /** \brief Syncronize execution of a routine with the required level * * This service guarantees execution of the routine won't overlap with execution * of the interrupt handler or the interrupt deferred handler. Also, this service * sets a corresponding execution level for the time the routine is being executed * * If unsupported value for sync_level is passed the routine is executed without * any level synchronization * * \param pSyncRoutine routine to run * \param pContext parameter to pass to the routine * \param sync_level execution level to sync the execution with * * \return Nonzero on success, zero on fail or unsupported sync_level value * */ unsigned int KAS_ExecuteAtLevel(void* pSyncRoutine, void* pContext, unsigned long sync_level) { unsigned long flags; unsigned long orig_level = kasContext.exec_level_invalid; unsigned int ret = 0; if (sync_level == kasContext.exec_level_idh) { spin_lock_bh(&kasContext.lock_idh); orig_level = kasSetExecutionLevel(kasContext.exec_level_idh); } else if (sync_level == kasContext.exec_level_ih) { spin_lock_irqsave(&kasContext.lock_ih, flags); // TODO: find out why compiler gives a warning here orig_level = kasSetExecutionLevel(kasContext.exec_level_ih); } ((kasSyncRoutine_t)pSyncRoutine)(pContext); if (sync_level == kasContext.exec_level_idh) { kasSetExecutionLevel(orig_level); spin_unlock_bh(&kasContext.lock_idh); ret = 1; } else if (sync_level == kasContext.exec_level_ih) { kasSetExecutionLevel(orig_level); spin_unlock_irqrestore(&kasContext.lock_ih, flags); ret = 1; } return ret; } /** \brief Type definition of the structure describing Spinlock object */ typedef struct tag_kasSpinlock_t { spinlock_t lock; /* OS spinlock object */ unsigned int routine_type; /* Type of routine the spinlock might be requested from */ unsigned int acquire_type; /* Type of OS spinlock function spinlock acquired with */ unsigned long flags; /* Saved CPU flags */ } kasSpinlock_t; /** \brief Return Spinlock object size * * \return Spinlock object size in bytes * */ unsigned int KAS_Spinlock_GetObjectSize(void) { return sizeof(kasSpinlock_t); } /** \brief Initialize Spinlock object * * \param hSpinLock handle of (pointer to) the Spinlock object * \param spinlock_type type of routine the spinlock might be requested from * * \return Nonzero (always success) * */ unsigned int KAS_Spinlock_Initialize(void* hSpinLock, unsigned int spinlock_type) { kasSpinlock_t* spinlock_obj = (kasSpinlock_t*)hSpinLock; spinlock_obj->acquire_type = KAS_SPINLOCK_TYPE_INVALID; spinlock_obj->routine_type = spinlock_type; spin_lock_init(&(spinlock_obj->lock)); return 1; } /** \brief Acquire Spinlock object * * \param hSpinLock handle of (pointer to) the Spinlock object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Spinlock_Acquire(void* hSpinLock) { unsigned int ret = 0; /* Fail by default */ kasSpinlock_t* spinlock_obj = (kasSpinlock_t*)hSpinLock; kas_spin_lock_info_t spin_lock_info; spin_lock_info.routine_type = spinlock_obj->routine_type; spin_lock_info.plock = &(spinlock_obj->lock); ret = kas_spin_lock(&spin_lock_info); spinlock_obj->acquire_type = spin_lock_info.acquire_type; spinlock_obj->flags = spin_lock_info.flags; return ret; } /** \brief Release Spinlock object * * \param hSpinLock handle of (pointer to) the Spinlock object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Spinlock_Release(void* hSpinLock) { unsigned int ret = 0; /* Fail by default */ kasSpinlock_t* spinlock_obj = (kasSpinlock_t*)hSpinLock; kas_spin_unlock_info_t spin_unlock_info; spin_unlock_info.plock = &(spinlock_obj->lock); spin_unlock_info.acquire_type = spinlock_obj->acquire_type; spin_unlock_info.flags = spinlock_obj->flags; if ((ret = kas_spin_unlock(&spin_unlock_info))) { spinlock_obj->acquire_type = KAS_SPINLOCK_TYPE_INVALID; } return ret; } /** \brief Type definition of the structure describing Slab Cache object */ typedef struct tag_kasSlabCache_t { kmem_cache_t* cache; /* OS slab cache object */ spinlock_t lock; /* OS spinlock object protecting the cache */ unsigned int routine_type; /* Type of routine the cache might be accessed from */ char name[24]; /* Cache object name */ } kasSlabCache_t; /** \brief Return Slab Cache object size * * \return Slab Cache object size in bytes * */ unsigned int KAS_SlabCache_GetObjectSize(void) { return sizeof(kasSlabCache_t); } /** \brief Initialize Slab Cache object * * \param hSlabCache handle of (pointer to) a Slab Cache object * \param iEntrySize size (in bytes) of each cache entry * \param access_type type of routine the spinlock might be requested from * * \return Nonzero on success, zero on fail * */ unsigned int KAS_SlabCache_Initialize(void* hSlabCache, unsigned int iEntrySize, unsigned int access_type) { kasSlabCache_t* slabcache_obj = (kasSlabCache_t*)hSlabCache; slabcache_obj->routine_type = access_type; spin_lock_init(&(slabcache_obj->lock)); sprintf(slabcache_obj->name, "slabcache_obj(%08X)", (unsigned int)slabcache_obj); return ((slabcache_obj->cache = kmem_cache_create(slabcache_obj->name, iEntrySize, 0, 0, NULL, NULL)) ? 1 : 0); } /** \brief Destroy Slab Cache object * * \param hSlabCache handle of (pointer to) a Slab Cache object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_SlabCache_Destroy(void* hSlabCache) { unsigned int ret = 0; kasSlabCache_t* slabcache_obj = (kasSlabCache_t*)hSlabCache; if (slabcache_obj->cache) { kmem_cache_destroy(slabcache_obj->cache); slabcache_obj->cache = NULL; ret = 1; } return ret; } /** \brief Allocate an entry in a Slab Cache * * \param hSlabCache handle of (pointer to) a Slab Cache object * * \return Pointer to the allocated entry (NULL indicates an error) * */ void* KAS_SlabCache_AllocEntry(void* hSlabCache) { kas_spin_lock_info_t spin_lock_info; kas_spin_unlock_info_t spin_unlock_info; void* pentry = NULL; kasSlabCache_t* slabcache_obj = (kasSlabCache_t*)hSlabCache; int alloc_flags = 0; /* Protect the operation with spinlock */ spin_lock_info.routine_type = slabcache_obj->routine_type; spin_lock_info.plock = &(slabcache_obj->lock); if (!kas_spin_lock(&spin_lock_info)) { return NULL; /* No spinlock - no operation */ } /* Allocate an entry */ if (kas_GetExecutionLevel() == kasContext.exec_level_regular) { alloc_flags = GFP_ATOMIC; } pentry = kmem_cache_alloc(slabcache_obj->cache, alloc_flags); /* Release the spinlock */ spin_unlock_info.plock = &(slabcache_obj->lock); spin_unlock_info.acquire_type = spin_lock_info.acquire_type; spin_unlock_info.flags = spin_lock_info.flags; if (!kas_spin_unlock(&spin_unlock_info)) { /* Signal an error if there were troubles releasing the spinlock */ kmem_cache_free(slabcache_obj->cache, pentry); pentry = NULL; } /* Return pointer to the allocated entry */ return pentry; } /** \brief Release an entry from a Slab Cache * * \param hSlabCache handle of (pointer to) a Slab Cache object * \param pvEntry pointer to the entry to be released * * \return Nonzero on success, zero on fail * */ unsigned int KAS_SlabCache_FreeEntry(void* hSlabCache, void* pvEntry) { kas_spin_lock_info_t spin_lock_info; kas_spin_unlock_info_t spin_unlock_info; kasSlabCache_t* slabcache_obj = (kasSlabCache_t*)hSlabCache; /* Protect the operation with spinlock */ spin_lock_info.routine_type = slabcache_obj->routine_type; spin_lock_info.plock = &(slabcache_obj->lock); if (!kas_spin_lock(&spin_lock_info)) { /* No spinlock - no operation (better to fail the release than to * deal with race condition on the cache object) */ return 0; } /* Release the entry */ kmem_cache_free(slabcache_obj->cache, pvEntry); /* Release the spinlock and return */ spin_unlock_info.plock = &(slabcache_obj->lock); spin_unlock_info.acquire_type = spin_lock_info.acquire_type; spin_unlock_info.flags = spin_lock_info.flags; return kas_spin_unlock(&spin_unlock_info); } /** \brief Type definition of the structure describing Event object */ typedef struct tag_kasEvent_t { wait_queue_head_t wq_head; atomic_t state; atomic_t wait_count; } kasEvent_t; /** \brief Return Event object size * * \return Event object size in bytes * */ unsigned int KAS_Event_GetObjectSize(void) { return sizeof(kasEvent_t); } /** \brief Initialize Event object * * \param hEvent handle of (pointer to) an Event object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Event_Initialize(void* hEvent) { kasEvent_t* event_obj = (kasEvent_t*)hEvent; init_waitqueue_head(&(event_obj->wq_head)); atomic_set(&(event_obj->state), 0); atomic_set(&(event_obj->wait_count), 0); return 1; } /** \brief Set event to the signalled state and wake up all waiters * * The event stays in the signalled state until cleared explicitly * * \param hEvent handle of (pointer to) an Event object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Event_Set(void* hEvent) { kasEvent_t* event_obj = (kasEvent_t*)hEvent; atomic_set(&(event_obj->state), 1); // Wake up all waiters until waiter count is not zero while (atomic_read(&(event_obj->wait_count))) { wake_up_all(&(event_obj->wq_head)); schedule(); } return 1; } /** \brief Clear the event (set it to the non-signalled state) * * \param hEvent handle of (pointer to) an Event object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Event_Clear(void* hEvent) { kasEvent_t* event_obj = (kasEvent_t*)hEvent; atomic_set(&(event_obj->state), 0); return 1; } /** \brief Wait for the event * * If event is already signalled, return right away. * Otherwise, wait until it is signalled * * For compatibility with kernel 2.4, interruptible_sleep_on_*() functions * are used. However, they may be deprecated in future 2.6 kernels, so that * these code might require kernel-dependent implementation * * \param hEvent handle of (pointer to) an Event object * \param timeout timeout value in nanoseconds * \param timeout_use 1 means wait with timeout, 0 means wait unconditionally * * \return KAS_RETCODE_OK on success * KAS_RETCODE_ERROR on error * KAS_RETCODE_TIMEOUT on timeout * */ unsigned int KAS_Event_WaitForEvent(void* hEvent, unsigned long long timeout, unsigned int timeout_use) { unsigned int ret = KAS_RETCODE_ERROR; kasEvent_t* event_obj = (kasEvent_t*)hEvent; if (atomic_read(&(event_obj->state))) { return KAS_RETCODE_OK; } atomic_inc(&(event_obj->wait_count)); if (timeout_use) { unsigned long long timeout_jiffies_ull = (timeout * HZ) / 1000000000; if (timeout_jiffies_ull <= 0x7FFFFFFF) { long timeout_jiffies = (long) timeout_jiffies_ull; timeout_jiffies = interruptible_sleep_on_timeout( &(event_obj->wq_head), timeout_jiffies); if (timeout_jiffies <= 0) { ret = KAS_RETCODE_TIMEOUT; } } } else { interruptible_sleep_on(&(event_obj->wq_head)); ret = KAS_RETCODE_OK; } atomic_dec(&(event_obj->wait_count)); return ret; } /** \brief Type definition of the structure describing Mutex object */ typedef struct tag_kasMutex_t { struct semaphore mutex; } kasMutex_t; /** \brief Return Mutex object size * * \return Mutex object size in bytes * */ unsigned int KAS_Mutex_GetObjectSize(void) { return sizeof(kasMutex_t); } /** \brief Initialize Mutex object * * \param hMutex handle of (pointer to) the Mutex object * * \return Nonzero (always success) * */ unsigned int KAS_Mutex_Initialize(void* hMutex) { kasMutex_t* mutex_obj = (kasMutex_t*)hMutex; init_MUTEX(&(mutex_obj->mutex)); return 1; } /** \brief Acquire Mutex object * * \param hMutex handle of (pointer to) the Mutex object * \param timeout timeout value in nanoseconds * \param timeout_use 1 means wait with timeout, 0 means wait unconditionally * * \return KAS_RETCODE_OK on success * KAS_RETCODE_ERROR on error * KAS_RETCODE_TIMEOUT on timeout * */ unsigned int KAS_Mutex_Acquire(void* hMutex, unsigned long long timeout, unsigned int timeout_use) { unsigned int ret = KAS_RETCODE_ERROR; kasMutex_t* mutex_obj = (kasMutex_t*)hMutex; if (timeout_use) { unsigned long long timeout_jiffies_ull = (timeout * HZ) / 1000000000; if (timeout_jiffies_ull <= 0x7FFFFFFF) { unsigned long jiffies_expire = jiffies + (unsigned long) timeout_jiffies_ull; while (time_before(jiffies, jiffies_expire)) { if (down_trylock(&(mutex_obj->mutex)) == 0) { ret = KAS_RETCODE_OK; } schedule(); } ret = KAS_RETCODE_TIMEOUT; } } else { if (down_interruptible(&(mutex_obj->mutex)) == 0) { ret = KAS_RETCODE_OK; } } return ret; } /** \brief Release Mutex object * * \param hMutex handle of (pointer to) the Mutex object * * \return Nonzero on success, zero on fail * */ unsigned int KAS_Mutex_Release(void* hMutex) { kasMutex_t* mutex_obj = (kasMutex_t*)hMutex; up(&(mutex_obj->mutex)); return 1; } /** \brief Type definition of the structure describing Thread object * * Thread object must be used in the following scenario only: * * 1) Create a Thread object * 2) Start the Thread object * 3) Tell the thread routine it has to finish. The thread routine has to signal * after it finishes * 4) Issue "wait for finish" operation * * Don't try to reuse a Thread object or issue multiple start operation on the same * object * */ #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0) #define work_struct tq_struct #define INIT_WORK INIT_TQUEUE #define schedule_work schedule_task #endif /** \brief Type definition of the structure describing Thread object */ typedef struct tag_kasThread_t { struct work_struct work; wait_queue_head_t wq_head; struct semaphore sleep_finished; } kasThread_t; /** \brief Return Thread object size * * \return Thread object size in bytes * */ unsigned int KAS_Thread_GetObjectSize(void) { return sizeof(kasThread_t); } /** \brief Start Thread * * \param hThread handle of (pointer to) a Thread object * \param routine pointer to a thread routine * \param pcontext context pointer to be passed to the thread routine * * \return Nonzero (always success) * */ unsigned int KAS_Thread_Start(void* hThread, KAS_ThreadRoutine_t routine, void* pcontext) { kasThread_t* thread_obj = (kasThread_t*)hThread; init_MUTEX(&(thread_obj->sleep_finished)); init_waitqueue_head(&(thread_obj->wq_head)); INIT_WORK(&(thread_obj->work), routine); schedule_work(&(thread_obj->work)); return 1; } /** \brief Wait until thread routine signals it finished * * \param hThread handle of (pointer to) a Thread object * * \return Nonzero (always success) * */ unsigned int KAS_Thread_WaitForFinish(void* hThread) { kasThread_t* thread_obj = (kasThread_t*)hThread; interruptible_sleep_on(&(thread_obj->wq_head)); up(&(thread_obj->sleep_finished)); return 1; } /** \brief Signal the thread finished its code path * * \param hThread handle of (pointer to) a Thread object * * \return Nonzero (always success) * */ unsigned int KAS_Thread_Finish(void* hThread) { kasThread_t* thread_obj = (kasThread_t*)hThread; down(&(thread_obj->sleep_finished)); // Wake up the waiter until it acks while (down_trylock(&(thread_obj->sleep_finished)) != 0) { wake_up(&(thread_obj->wq_head)); schedule(); } up(&(thread_obj->sleep_finished)); return 1; } /** \brief Type definition of the structure describing InterlockedList head object */ typedef struct tag_kasInterlockedListHead_t { struct list_head head; spinlock_t lock; unsigned int routine_type; /* Type of routine the list might be accessed from */ } kasInterlockedListHead_t; /** \brief Type definition of the structure describing InterlockedList entry object */ typedef struct tag_kasInterlockedListEntry_t { struct list_head entry; } kasInterlockedListEntry_t; /** \brief Return InterlockedList head object size * * \return InterlockedList head object size in bytes * */ unsigned int KAS_InterlockedList_GetListHeadSize(void) { return sizeof(kasInterlockedListHead_t); } /** \brief Return InterlockedList entry object size * * \return InterlockedList entry object size in bytes * */ unsigned int KAS_InterlockedList_GetListEntrySize(void) { return sizeof(kasInterlockedListEntry_t); } /** \brief Initialize InterlockedList object * * \param hListHead handle of (pointer to) an InterlockedList object * * \return Nonzero (always success) * */ unsigned int KAS_InterlockedList_Initialize(void* hListHead, unsigned int access_type) { kasInterlockedListHead_t* listhead_obj = (kasInterlockedListHead_t*)hListHead; INIT_LIST_HEAD(&(listhead_obj->head)); spin_lock_init(&(listhead_obj->lock)); listhead_obj->routine_type = access_type; return 1; } /** \brief Insert an entry at the tail of a list * * \param hListHead handle of (pointer to) an InterlockedList head object * \param hListEntry handle of (pointer to) an InterlockedList entry object * \param phPrevEntry pointer to the handle of (pointer to) the previous tail entry * * \return Nonzero on success, zero on fail * */ unsigned int KAS_InterlockedList_InsertAtTail(void* hListHead, void* hListEntry, void** phPrevEntry) { kasInterlockedListHead_t* listhead_obj = (kasInterlockedListHead_t*)hListHead; kasInterlockedListEntry_t* listentry_obj = (kasInterlockedListEntry_t*)hListEntry; struct list_head *head = &(listhead_obj->head); struct list_head *entry = &(listentry_obj->entry); kas_spin_lock_info_t spin_lock_info; kas_spin_unlock_info_t spin_unlock_info; /* Protect the operation with spinlock */ spin_lock_info.routine_type = listhead_obj->routine_type; spin_lock_info.plock = &(listhead_obj->lock); if (!kas_spin_lock(&spin_lock_info)) { return 0; /* No spinlock - no operation */ } /* Get pointer to the current tail entry */ if (list_empty(head)) { *phPrevEntry = NULL; } else { *phPrevEntry = list_entry(head->prev, kasInterlockedListEntry_t, entry); } /* Add the new entry to the tail of the list */ list_add_tail(entry, head); /* Release the spinlock and return */ spin_unlock_info.plock = &(listhead_obj->lock); spin_unlock_info.acquire_type = spin_lock_info.acquire_type; spin_unlock_info.flags = spin_lock_info.flags; return kas_spin_unlock(&spin_unlock_info); } /** \brief Insert an entry at the head of a list * * \param hListHead handle of (pointer to) an InterlockedList head object * \param hListEntry handle of (pointer to) an InterlockedList entry object * \param phPrevEntry pointer to the handle of (pointer to) the previous head entry * * \return Nonzero on success, zero on fail * */ unsigned int KAS_InterlockedList_InsertAtHead(void* hListHead, void* hListEntry, void** phPrevEntry) { kasInterlockedListHead_t* listhead_obj = (kasInterlockedListHead_t*)hListHead; kasInterlockedListEntry_t* listentry_obj = (kasInterlockedListEntry_t*)hListEntry; struct list_head *head = &(listhead_obj->head); struct list_head *entry = &(listentry_obj->entry); kas_spin_lock_info_t spin_lock_info; kas_spin_unlock_info_t spin_unlock_info; /* Protect the operation with spinlock */ spin_lock_info.routine_type = listhead_obj->routine_type; spin_lock_info.plock = &(listhead_obj->lock); if (!kas_spin_lock(&spin_lock_info)) { return 0; /* No spinlock - no operation */ } /* Get pointer to the current head entry */ if (list_empty(head)) { *phPrevEntry = NULL; } else { *phPrevEntry = list_entry(head->next, kasInterlockedListEntry_t, entry); } /* Add the new entry to the beginning of the list */ list_add(entry, head); /* Release the spinlock and return */ spin_unlock_info.plock = &(listhead_obj->lock); spin_unlock_info.acquire_type = spin_lock_info.acquire_type; spin_unlock_info.flags = spin_lock_info.flags; return kas_spin_unlock(&spin_unlock_info); } /** \brief Remove the head entry from a list * * \param hListHead handle of (pointer to) an InterlockedList head object * \param phRemovedEntry pointer to the handle of (pointer to) the removed entry * * \return Nonzero on success, zero on fail * */ unsigned int KAS_InterlockedList_RemoveAtHead(void* hListHead, void** phRemovedEntry) { kasInterlockedListHead_t* listhead_obj = (kasInterlockedListHead_t*)hListHead; struct list_head *head = &(listhead_obj->head); kas_spin_lock_info_t spin_lock_info; kas_spin_unlock_info_t spin_unlock_info; /* Protect the operation with spinlock */ spin_lock_info.routine_type = listhead_obj->routine_type; spin_lock_info.plock = &(listhead_obj->lock); if (!kas_spin_lock(&spin_lock_info)) { return 0; /* No spinlock - no operation */ } /* Get pointer to the current head entry */ if (list_empty(head)) { *phRemovedEntry = NULL; } else { *phRemovedEntry = list_entry(head->next, kasInterlockedListEntry_t, entry); } /* Add the new entry to the beginning of the list */ list_del(head->next); /* Release the spinlock and return */ spin_unlock_info.plock = &(listhead_obj->lock); spin_unlock_info.acquire_type = spin_lock_info.acquire_type; spin_unlock_info.flags = spin_lock_info.flags; return kas_spin_unlock(&spin_unlock_info); } /** \brief Atomic compare and exchange operation for unsigned int * * If uiComparand is equal to *puiDestination, * then *puiDestination is set equal to uiExchange. * Otherwise, *puiDestination is unchanged. * * \param puiDestination Pointer to the destination operand * \param uiExchange Source operand * \param uiComparand Value to compare * * \return Old value of *puiDestination. */ unsigned int KAS_AtomicCompareExchangeUnsignedInt( unsigned int *puiDestination, unsigned int uiExchange, unsigned int uiComparand) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT return cmpxchg(puiDestination, uiComparand, uiExchange); #else return 0xDEADC0DE; /* To make compiler happy */ #endif } /** \brief Atomic exchange operation for unsigned int * * *puiDestination is set equal to uiExchange. * * \param puiDestination Pointer to the destination operand * \param uiExchange Source operand * * \return Old value of *puiDestination. */ unsigned int KAS_AtomicExchangeUnsignedInt( unsigned int *puiDestination, unsigned int uiExchange) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT return xchg(puiDestination, uiExchange); #else return 0xDEADC0DE; /* To make compiler happy */ #endif } /** \brief Definition for assembly lock prefix */ #ifdef CONFIG_SMP #define KAS_LOCK_PREFIX "lock ; " #else #define KAS_LOCK_PREFIX "" #endif /** \brief Macro for assembly XADD operation */ #define kas_xadd(dest,add,ret,size) \ __asm__ __volatile__( \ KAS_LOCK_PREFIX \ "xadd" size " %0,(%1)" \ : "=r" (ret) \ : "r" (dest), "0" (add) \ ); /** \brief Atomic exchange and add operation for unsigned int * * Add uiAdd to *puiDestination * * \param puiDestination Pointer to the destination operand * \param uiAdd Value to add * * \return Old value of *puiDestination */ unsigned int KAS_AtomicExchangeAddUnsignedInt( unsigned int *puiDestination, unsigned int uiAdd) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT unsigned int ret; int size_ui = sizeof(unsigned int); switch (size_ui) { case 4: kas_xadd(puiDestination, uiAdd, ret, "l"); break; case 8: kas_xadd(puiDestination, uiAdd, ret, "q"); break; default: kas_xadd(puiDestination, uiAdd, ret, ""); break; } return ret; #else return 0xDEADC0DE; /* To make compiler happy */ #endif } /** \brief Atomic add operation for unsigned int * * Add iAdd to *puiDestination. iAdd can be negative, to perform atomic * substructions * * \param puiDestination Pointer to the destination operand * \param iAdd value to add * * \return New value of *puiDestination * */ unsigned int KAS_AtomicAddInt(unsigned int *puiDestination, int iAdd) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT unsigned int ret; int size_ui = sizeof(unsigned int); switch (size_ui) { case 4: kas_xadd(puiDestination, iAdd, ret, "l"); break; case 8: kas_xadd(puiDestination, iAdd, ret, "q"); break; default: kas_xadd(puiDestination, iAdd, ret, ""); break; } return ret + iAdd; #else return 0xDEADC0DE; /* To make compiler happy */ #endif } /** \brief Atomic compare and exchange operation for pointers * * If pvComparand is equal to *ppvDestination, * then *ppvDestination is set equal to pvExchange. * Otherwise, *ppvDestination is unchanged. * * \param ppvDestination Pointer to the destination operand * \param pvExchange Source operand * \param pvComparand Value to compare * * \return Old value of *ppvDestination. */ void* KAS_AtomicCompareExchangePointer( void* *ppvDestination, void* pvExchange, void* pvComparand) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT return cmpxchg(ppvDestination, pvComparand, pvExchange); #else return 0xDEADC0DE; /* To make compiler happy */ #endif } /** \brief Atomic exchange operation for pointers * * *ppvDestination is set equal to pvExchange. * * \param ppvDestination Pointer to the destination operand * \param pvExchange Source operand * * \return Old value of *ppvDestination. */ void* KAS_AtomicExchangePointer( void* *ppvDestination, void* pvExchange) { #ifdef KAS_ATOMIC_OPERATIONS_SUPPORT return xchg(ppvDestination, pvExchange); #else return 0xDEADC0DE; /* To make compiler happy */ #endif } #endif /* __KERNEL__ */