{"id":1362,"date":"2016-03-16T21:45:23","date_gmt":"2016-03-16T12:45:23","guid":{"rendered":"http:\/\/avinton.com\/?p=1362"},"modified":"2020-02-27T16:24:08","modified_gmt":"2020-02-27T07:24:08","slug":"big-data-solutions-storage","status":"publish","type":"post","link":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/","title":{"rendered":"Big Data Solutions &#8211; Storage"},"content":{"rendered":"<div class=\"wpb-content-wrapper\"><p>[vc_row][vc_column][vc_column_text]<\/p>\n<h2>Considerations<\/h2>\n<p>[\/vc_column_text][\/vc_column][\/vc_row][vc_row type=&#8221;full-width-section&#8221; css=&#8221;.vc_custom_1444119952127{margin-bottom: 0px !important;border-top-width: 1px !important;padding-top: 50px !important;border-top-color: #ebebeb !important;border-top-style: solid !important;}&#8221;][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562859265{margin-bottom: 20px !important;}&#8221;]<style type=\"text\/css\" >#rand_rf9p9luf01xo5wwj1488 {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'ln'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ln.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ln.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ln.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ln.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ln.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_rf9p9luf01xo5wwj1488 .ln-lock:before{font-family: 'ln'; font-weight: normal; font-style: normal;}#rand_rf9p9luf01xo5wwj1488 .ln-lock:before{content: \"\\e00d\";}<\/style>\n<div id=\"rand_rf9p9luf01xo5wwj1488\" class=\"icon_box_big_rounded_stroke \"><i class=\"ln-lock\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1C2833;\">Security<\/h3><p style=\"color:#a1b1bc\">Using hardened APIs and best practice security measures corresponding to the environment.<\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243;]<style type=\"text\/css\" >#rand_tlq4efwwxh58zeardwti {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'imf'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_tlq4efwwxh58zeardwti .imf-shield:before{font-family: 'imf'; font-weight: normal; font-style: normal;}#rand_tlq4efwwxh58zeardwti .imf-shield:before{content: \"\\e27e\";}<\/style>\n<div id=\"rand_tlq4efwwxh58zeardwti\" class=\"icon_box_big_rounded_stroke \"><i class=\"imf-shield\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1C2833;\">Reliability<\/h3><p style=\"color:#a1b1bc\">The Data Store is required to respond consistently to Read and Write requests.<\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243;]<style type=\"text\/css\" >#rand_v1qspfpujr54d084t3m1 {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'mt'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_v1qspfpujr54d084t3m1 .mt-windy-2:before{font-family: 'mt'; font-weight: normal; font-style: normal;}#rand_v1qspfpujr54d084t3m1 .mt-windy-2:before{content: \"\\e012\";}<\/style>\n<div id=\"rand_v1qspfpujr54d084t3m1\" class=\"icon_box_big_rounded_stroke \"><i class=\"mt-windy-2\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1c2833;\">Speed!<\/h3><p style=\"color:#a1b1bc\">The storage solution needs\u00a0to accommodate IO operations at a speed that the application demands to maintain user experience and data concurrency<\/p><\/div>[\/vc_column][\/vc_row][vc_row type=&#8221;full-width-section&#8221; css=&#8221;.vc_custom_1444119952127{margin-bottom: 0px !important;border-top-width: 1px !important;padding-top: 50px !important;border-top-color: #ebebeb !important;border-top-style: solid !important;}&#8221;][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562859265{margin-bottom: 20px !important;}&#8221;]<style type=\"text\/css\" >#rand_zb0obhft4of1co8s5m89 {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'ls'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ls.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ls.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ls.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ls.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/ls.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_zb0obhft4of1co8s5m89 .ls-graph:before{font-family: 'ls'; font-weight: normal; font-style: normal;}#rand_zb0obhft4of1co8s5m89 .ls-graph:before{content: \"\\e84b\";}<\/style>\n<div id=\"rand_zb0obhft4of1co8s5m89\" class=\"icon_box_big_rounded_stroke \"><i class=\"ls-graph\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1c2833;\">Scalability<\/h3><p style=\"color:#a1b1bc\">An appropriate storage solution selection (ex: DAS \/ SAN \/ NAS) which can scale well into the future demands of the application and within the environment constraints.<\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562879961{margin-bottom: 20px !important;}&#8221;]<style type=\"text\/css\" >#rand_szhdi8fafhoqmzj4mkg4 {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'wb'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_szhdi8fafhoqmzj4mkg4 .wb-copy:before{font-family: 'wb'; font-weight: normal; font-style: normal;}#rand_szhdi8fafhoqmzj4mkg4 .wb-copy:before{content: \"\\f0c9\";}<\/style>\n<div id=\"rand_szhdi8fafhoqmzj4mkg4\" class=\"icon_box_big_rounded_stroke \"><i class=\"wb-copy\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1c2833;\">Backup<\/h3><p style=\"color:#a1b1bc\">A Robust backup solution design for\u00a0minimal HW footprint with appropriate point in time recovery, system\u00a0restore time, geo-redundancy and replication factor.<\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562884787{margin-bottom: 20px !important;}&#8221;]<style type=\"text\/css\" >#rand_biwwe4xcsms1xbxu6w9l {margin-top:0px; margin-bottom:0px; }<\/style><style type='text\/css'> @font-face{font-family: 'wb'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wb.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_biwwe4xcsms1xbxu6w9l .wb-tank:before{font-family: 'wb'; font-weight: normal; font-style: normal;}#rand_biwwe4xcsms1xbxu6w9l .wb-tank:before{content: \"\\f423\";}<\/style>\n<div id=\"rand_biwwe4xcsms1xbxu6w9l\" class=\"icon_box_big_rounded_stroke \"><i class=\"wb-tank\" style=\"border:1px solid #dd3333; color:#dd3333;\"><\/i><h3 style=\"color:#1c2833;\">Fault Tolerance<\/h3><p style=\"color:#a1b1bc\">Rugged fault tolerance built into both the hardware and software layer.<\/p><\/div>[\/vc_column][\/vc_row][vc_row][vc_column][vc_column_text]<\/p>\n<h2>How can we achieve this?<\/h2>\n<p>&nbsp;<\/p>\n<div>\n<h3>\u00a0&#8211;\u00a0Storage Path Optimisation<\/h3>\n<p>&nbsp;<\/p>\n<p>Keep the data as close as possible to where it\u2019s being used<\/p>\n<p>&nbsp;<\/p>\n<\/div>\n<h3>\u00a0&#8211; Accelerated Data Access<\/h3>\n<ul>\n<li>Multidimensional Caching\n<ul>\n<li>Hardware Level (ex. Storage Controller Cache \/ SAN Cache Pool)<\/li>\n<li>Storage Fabric Level<\/li>\n<li>System Service Level<\/li>\n<li>Application Data Access Layer \/ API Level<\/li>\n<\/ul>\n<\/li>\n<li>In-Memory Datasets and Indexes<\/li>\n<li>Adaptive Data Compression, De-duplication, Preallocation<\/li>\n<\/ul>\n<p>&nbsp;<\/p>\n<h3><b>&#8211; Data Classification for performance and cost efficiency\u00a0<\/b><\/h3>\n<ul>\n<li>From simple data access frequency or age based to complex, pattern based or statistical predictive algorithms,<\/li>\n<li>Or\u00a0Data Type based classification for Object, Block and File Storage<\/li>\n<li>Example:<\/li>\n<\/ul>\n<h3><\/h3>\n<p>[\/vc_column_text][\/vc_column][\/vc_row][vc_row][vc_column width=&#8221;1\/3&#8243;]<style type=\"text\/css\" >#rand_xhhc79eanics88xr6t9f{border:1px solid #ecf0f1; background:#ffffff; }#rand_xhhc79eanics88xr6t9f .icon_circle{border:1px solid #ecf0f1; background:#ffffff; color:#b50000;}#rand_xhhc79eanics88xr6t9f .icon_sub_ctn{border:1px solid #ecf0f1; background:#ffffff; }<\/style> <style type='text\/css'> @font-face{font-family: 'zm'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/zm.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/zm.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/zm.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/zm.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/zm.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_xhhc79eanics88xr6t9f .zm-lightning:before{font-family: 'zm'; font-weight: normal; font-style: normal;}#rand_xhhc79eanics88xr6t9f .zm-lightning:before{content: \"\\e07c\";}<\/style>\n<div id=\"rand_xhhc79eanics88xr6t9f\" class=\"icon_box2_st \"><div class=\"icon_circle\"><div class=\"icon_sub_ctn\"><i class=\"zm-lightning\" ><\/i><\/div><\/div><h3 style=\"color:;\" >Hot Data ~ In Memory<\/h3><p style=\"color:;\" ><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Extremely high frequency<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Semi-structured transactional data<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Preprocessing subsets<\/span><\/p>\n<p><\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243;]<style type=\"text\/css\" >#rand_51xx8ar146s6e8cn0b6y{border:1px solid #ecf0f1; background:#ffffff; }#rand_51xx8ar146s6e8cn0b6y .icon_circle{border:1px solid #ecf0f1; background:#ffffff; color:#b50000;}#rand_51xx8ar146s6e8cn0b6y .icon_sub_ctn{border:1px solid #ecf0f1; background:#ffffff; }<\/style> <style type='text\/css'> @font-face{font-family: 'wi'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_51xx8ar146s6e8cn0b6y .wi-sunrise-sunset-1:before{font-family: 'wi'; font-weight: normal; font-style: normal;}#rand_51xx8ar146s6e8cn0b6y .wi-sunrise-sunset-1:before{content: \"\\e843\";}<\/style>\n<div id=\"rand_51xx8ar146s6e8cn0b6y\" class=\"icon_box2_st \"><div class=\"icon_circle\"><div class=\"icon_sub_ctn\"><i class=\"wi-sunrise-sunset-1\" ><\/i><\/div><\/div><h3 style=\"color:;\" >Warm Data ~ Flash Disk (SSD)<\/h3><p style=\"color:;\" ><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">High Frequency<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Structured Subsets<\/span><\/p>\n<p><\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/3&#8243;]<style type=\"text\/css\" >#rand_lcc5lv8rrcu6wi7coz9b{border:1px solid #ecf0f1; background:#ffffff; }#rand_lcc5lv8rrcu6wi7coz9b .icon_circle{border:1px solid #ecf0f1; background:#ffffff; color:#b50000;}#rand_lcc5lv8rrcu6wi7coz9b .icon_sub_ctn{border:1px solid #ecf0f1; background:#ffffff; }<\/style> <style type='text\/css'> @font-face{font-family: 'mt'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack04\/icons\/fonts\/mt.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_lcc5lv8rrcu6wi7coz9b .mt-windy-5:before{font-family: 'mt'; font-weight: normal; font-style: normal;}#rand_lcc5lv8rrcu6wi7coz9b .mt-windy-5:before{content: \"\\e024\";}<\/style>\n<div id=\"rand_lcc5lv8rrcu6wi7coz9b\" class=\"icon_box2_st \"><div class=\"icon_circle\"><div class=\"icon_sub_ctn\"><i class=\"mt-windy-5\" ><\/i><\/div><\/div><h3 style=\"color:;\" >Cold Data ~ Fast Disk Array<\/h3><p style=\"color:;\" ><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Low\u00a0Frequency<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Structured Subsets<\/span><\/p>\n<p><\/p><\/div>[\/vc_column][\/vc_row][vc_row][vc_column width=&#8221;1\/2&#8243;]<style type=\"text\/css\" >#rand_f5cqb10alfxhlxkhd27e{border:1px solid #ecf0f1; background:#ffffff; }#rand_f5cqb10alfxhlxkhd27e .icon_circle{border:1px solid #ecf0f1; background:#ffffff; color:#b50000;}#rand_f5cqb10alfxhlxkhd27e .icon_sub_ctn{border:1px solid #ecf0f1; background:#ffffff; }<\/style> <style type='text\/css'> @font-face{font-family: 'wi'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack05\/icons\/fonts\/wi.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_f5cqb10alfxhlxkhd27e .wi-snowy-night:before{font-family: 'wi'; font-weight: normal; font-style: normal;}#rand_f5cqb10alfxhlxkhd27e .wi-snowy-night:before{content: \"\\e836\";}<\/style>\n<div id=\"rand_f5cqb10alfxhlxkhd27e\" class=\"icon_box2_st \"><div class=\"icon_circle\"><div class=\"icon_sub_ctn\"><i class=\"wi-snowy-night\" ><\/i><\/div><\/div><h3 style=\"color:;\" >ICY Data ~ Slow Disk Array \/ Remote<\/h3><p style=\"color:;\" ><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Very low frequency<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Structured<\/span><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Compressed Subsets<\/span><\/p>\n<p><\/p><\/div>[\/vc_column][vc_column width=&#8221;1\/2&#8243;]<style type=\"text\/css\" >#rand_vxw8hgo4z4lpua3rbjcq{border:1px solid #ecf0f1; background:#ffffff; }#rand_vxw8hgo4z4lpua3rbjcq .icon_circle{border:1px solid #ecf0f1; background:#ffffff; color:#b50000;}#rand_vxw8hgo4z4lpua3rbjcq .icon_sub_ctn{border:1px solid #ecf0f1; background:#ffffff; }<\/style> <style type='text\/css'> @font-face{font-family: 'imf'; src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.eot'); src:url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.eot?#iefix') format('embedded-opentype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.woff') format('woff'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.ttf') format('truetype'), url('https:\/\/avinton.com\/wp-content\/themes\/thefox\/includes\/4k-icons\/icons\/4k-icons-pack03\/icons\/fonts\/imf.svg#oi') format('svg'); font-weight: normal; font-style: normal;}#rand_vxw8hgo4z4lpua3rbjcq .imf-snowflake:before{font-family: 'imf'; font-weight: normal; font-style: normal;}#rand_vxw8hgo4z4lpua3rbjcq .imf-snowflake:before{content: \"\\e2e9\";}<\/style>\n<div id=\"rand_vxw8hgo4z4lpua3rbjcq\" class=\"icon_box2_st \"><div class=\"icon_circle\"><div class=\"icon_sub_ctn\"><i class=\"imf-snowflake\" ><\/i><\/div><\/div><h3 style=\"color:;\" >Frozen Data ~ Tape Library<\/h3><p style=\"color:;\" ><\/p>\n<p style=\"text-align: center;\"><span style=\"color: #808080;\">Archive<\/span><\/p>\n<p><\/p><\/div>[\/vc_column][\/vc_row][vc_row][vc_column][vc_column_text]It is necessary to identify differences in access patterns to the various pieces of data in order to ensure that the appropriate storage solution is chosen for each type.<br \/>\nData which is accessed or updated frequently can be classed as hot.<br \/>\nData which is accessed or updated occasionally can be classified as cold and warm being somewhere in between.<\/p>\n<p>These different classifications can allow us to further differentiate the replication factor and access speed required for the different data areas.<\/p>\n<p>&nbsp;<\/p>\n<h3>\u00a0&#8211; Reduce Disk IO needed for each data request<\/h3>\n<p>In case of BLOB (binary large object) storage we can use large volumes (~100GB) with an in-memory index.<br \/>\nThe 100GB Volume can store a number of images say with their respective location in the volume known by the index which is held in the storage node&#8217;s memory for quick access.<\/p>\n<p>&nbsp;<\/p>\n<h3>\u00a0&#8211; Using SSDs<\/h3>\n<p>At Avinton we design our solutions where we place the HOT data on SSD arrays and the cold data on the spinning disks. In cases where it is not immediately apparent which data is hot or cold we gather meta data on the files or tables in order to understand the number of reads, updates, index scans and so on which will then allow us to isolate the hot data.<br \/>\nIn some cases data classification (HOT \/ Warm \/ Cold) is relative to age so newer data will be HOT while the older data is expired onto the Cold storage area.<\/p>\n<p>&nbsp;<\/p>\n<h3>\u00a0&#8211; In-Memory Index<\/h3>\n<p>In scenarios where the data volumes are really large we also use in-memory indexes &#8211; typically in the form of key-value pairs. With the recent improvements in the reliability of in-memory key-value pair solutions with persistence we are able to achieve significant performance gains with minimal risk.<\/p>\n<p>&nbsp;<\/p>\n<h2>Conclusions<\/h2>\n<h3>\u00a0&#8211; Improved Application Performance<\/h3>\n<p>Done right the application of such techniques will improve the data response speed significantly and is often part of the solution for long running queries. In some cases we are able to improve the storage performance while avoiding a costly hardware upgrade.<\/p>\n<p>&nbsp;<\/p>\n<h3>\u00a0&#8211; Data Schema Considerations<\/h3>\n<p>In the case of any storage solution one cannot rely on these techniques alone. A good schema in the case of an RDBMS data warehouse is key for having a responsive solution. Other areas to look at are bottlenecks on the data input and output interfaces (be it SCSI \/ SAS \/ IP).<\/p>\n<p>&nbsp;<\/p>\n<h3>\u00a0&#8211; Why Avinton?<\/h3>\n<p>Avinton are by no means pioneers in this area &#8211; similar techniques are used by Google, Facebook, Yahoo and many other big players. We have simply mastered these techniques having been using them throughout the years starting from our early Telecom monitoring solutions which are still in use today.<\/p>\n<p>&nbsp;<\/p>\n<h2>Final Thoughts<\/h2>\n<p>To design a good data storage solution the following are necessary:<br \/>\n&#8211; Know your data (HOT vs Cold &#8211; Structure, Size, Types etc..)<br \/>\n&#8211; Know your users (#Simultaneous Users, Types of queries)<br \/>\n&#8211; Detailed knowledge of HW (Server vendor specific HW options)<br \/>\n&#8211; Good working knowledge of the storage technique in use (be it DB or File based storage)<br \/>\n&#8211; Appropriate Storage Solution Selection (DAS \/ SAN \/ NAS)<\/p>\n<p>Having a scalable Big Data Storage solution that allows you to leverage data insights efficiently is fundamental since having a lot of data which is slow to retrieve diminishes its value.<\/p>\n<p>Avinton have designed and delivered various data solutions including both RDBMS (PostgreSQL &amp; ORACLE) and hybrid RDBMS &amp; file based solutions on HDFS (Hadoop).<br \/>\nWe offer an End to End\u00a0service from Design &gt; Dimensioning &gt; Implementation &gt; Deployment &gt; SLA based Support.<\/p>\n<p>A\u00a0common theme throughout this article is that Avinton&#8217;s solutions feature design considerations for improved IO performance\u00a0both on the Software and Hardware level. This stems from our philosophy that to design high performance big data solutions one has to have a good understanding of the underlying hardware.<\/p>\n<p>Our Research, Development and Testing work at our development and training centre in Yokohama is where we test new hardware configurations\u00a0and combine them\u00a0with well known big data solutions like our latest project with Spark on Hadoop. This allows us to bring our clients tailored solutions based on test result data.<\/p>\n<p>Our Research, Development and Testing experience Enables\u00a0us to:<\/p>\n<ul>\n<li>Deliver optimised HW \/ SW platform combinations<\/li>\n<li>Reduce time to market<\/li>\n<li>Heavily Tailor the solution to our client&#8217;s design requirements<\/li>\n<li>Provide SLA based HW &amp; Application support<\/li>\n<\/ul>\n<p>We are passionate about data and welcome any <a href=\"http:\/\/avinton.com\/contact\/\">enquiries<\/a> in this regard.[\/vc_column_text][\/vc_column][\/vc_row]<\/p>\n<\/div>","protected":false},"excerpt":{"rendered":"<p>[vc_row][vc_column][vc_column_text] Considerations [\/vc_column_text][\/vc_column][\/vc_row][vc_row type=&#8221;full-width-section&#8221; css=&#8221;.vc_custom_1444119952127{margin-bottom: 0px !important;border-top-width: 1px !important;padding-top: 50px !important;border-top-color: #ebebeb !important;border-top-style: solid !important;}&#8221;][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562859265{margin-bottom: 20px !important;}&#8221;][\/vc_column][vc_column width=&#8221;1\/3&#8243;][\/vc_column][vc_column width=&#8221;1\/3&#8243;][\/vc_column][\/vc_row][vc_row type=&#8221;full-width-section&#8221; css=&#8221;.vc_custom_1444119952127{margin-bottom: 0px !important;border-top-width: 1px !important;padding-top: 50px !important;border-top-color: #ebebeb !important;border-top-style: solid !important;}&#8221;][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562859265{margin-bottom: 20px !important;}&#8221;][\/vc_column][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562879961{margin-bottom: 20px !important;}&#8221;][\/vc_column][vc_column width=&#8221;1\/3&#8243; css=&#8221;.vc_custom_1442562884787{margin-bottom: 20px !important;}&#8221;][\/vc_column][\/vc_row][vc_row][vc_column][vc_column_text] How can we achieve this? &nbsp; \u00a0&#8211;\u00a0Storage Path Optimisation &nbsp;<br \/><a href=\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\" class=\"more\">Read more<\/a><\/p>\n","protected":false},"author":2,"featured_media":1679,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[8,1580],"tags":[346,480],"class_list":["post-1362","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-tech-articles","category-mainpage","tag-infrastructure","tag-storage"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v25.8 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>Avinton Big Data Solutions - Storage<\/title>\n<meta name=\"description\" content=\"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Avinton Big Data Solutions - Storage\" \/>\n<meta property=\"og:description\" content=\"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..\" \/>\n<meta property=\"og:url\" content=\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\" \/>\n<meta property=\"og:site_name\" content=\"Avinton Japan\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/Avintons\/\" \/>\n<meta property=\"article:published_time\" content=\"2016-03-16T12:45:23+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2020-02-27T07:24:08+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg\" \/>\n\t<meta property=\"og:image:width\" content=\"1440\" \/>\n\t<meta property=\"og:image:height\" content=\"500\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/jpeg\" \/>\n<meta name=\"author\" content=\"James Cauchi\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:creator\" content=\"@AvintonJapan\" \/>\n<meta name=\"twitter:site\" content=\"@AvintonJapan\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"James Cauchi\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"7 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\",\"url\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\",\"name\":\"Avinton Big Data Solutions - Storage\",\"isPartOf\":{\"@id\":\"https:\/\/avinton.com\/en\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage\"},\"image\":{\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage\"},\"thumbnailUrl\":\"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg\",\"datePublished\":\"2016-03-16T12:45:23+00:00\",\"dateModified\":\"2020-02-27T07:24:08+00:00\",\"author\":{\"@id\":\"https:\/\/avinton.com\/en\/#\/schema\/person\/aa5bcc7a7c363ca85c0eeb6a7c2c594b\"},\"description\":\"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..\",\"breadcrumb\":{\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage\",\"url\":\"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg\",\"contentUrl\":\"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg\",\"width\":1440,\"height\":500,\"caption\":\"Avinton - Storage Solutions\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/avinton.com\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Big Data Solutions &#8211; Storage\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/avinton.com\/en\/#website\",\"url\":\"https:\/\/avinton.com\/en\/\",\"name\":\"Avinton Japan\",\"description\":\"Tailored Solutions and Consulting in AI and Big Data\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/avinton.com\/en\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/avinton.com\/en\/#\/schema\/person\/aa5bcc7a7c363ca85c0eeb6a7c2c594b\",\"name\":\"James Cauchi\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/avinton.com\/en\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/24fff15ecfe40a23480c47de1acb5c69cc3aa019d6f6cd36353cee85ac20a9e7?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/24fff15ecfe40a23480c47de1acb5c69cc3aa019d6f6cd36353cee85ac20a9e7?s=96&d=mm&r=g\",\"caption\":\"James Cauchi\"}}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Avinton Big Data Solutions - Storage","description":"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/","og_locale":"en_US","og_type":"article","og_title":"Avinton Big Data Solutions - Storage","og_description":"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..","og_url":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/","og_site_name":"Avinton Japan","article_publisher":"https:\/\/www.facebook.com\/Avintons\/","article_published_time":"2016-03-16T12:45:23+00:00","article_modified_time":"2020-02-27T07:24:08+00:00","og_image":[{"width":1440,"height":500,"url":"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg","type":"image\/jpeg"}],"author":"James Cauchi","twitter_card":"summary_large_image","twitter_creator":"@AvintonJapan","twitter_site":"@AvintonJapan","twitter_misc":{"Written by":"James Cauchi","Est. reading time":"7 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/","url":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/","name":"Avinton Big Data Solutions - Storage","isPartOf":{"@id":"https:\/\/avinton.com\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage"},"image":{"@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage"},"thumbnailUrl":"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg","datePublished":"2016-03-16T12:45:23+00:00","dateModified":"2020-02-27T07:24:08+00:00","author":{"@id":"https:\/\/avinton.com\/en\/#\/schema\/person\/aa5bcc7a7c363ca85c0eeb6a7c2c594b"},"description":"Big Data Solutions - Storage Considerations from Infrastructure, Storage Types to Data Design. Speed, Security, Backup, Data Schema, HW Footprint..","breadcrumb":{"@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#primaryimage","url":"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg","contentUrl":"https:\/\/avinton.com\/wp-content\/uploads\/2016\/03\/storsol.jpg","width":1440,"height":500,"caption":"Avinton - Storage Solutions"},{"@type":"BreadcrumbList","@id":"https:\/\/avinton.com\/en\/blog\/2016\/03\/big-data-solutions-storage\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/avinton.com\/en\/"},{"@type":"ListItem","position":2,"name":"Big Data Solutions &#8211; Storage"}]},{"@type":"WebSite","@id":"https:\/\/avinton.com\/en\/#website","url":"https:\/\/avinton.com\/en\/","name":"Avinton Japan","description":"Tailored Solutions and Consulting in AI and Big Data","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/avinton.com\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Person","@id":"https:\/\/avinton.com\/en\/#\/schema\/person\/aa5bcc7a7c363ca85c0eeb6a7c2c594b","name":"James Cauchi","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/avinton.com\/en\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/24fff15ecfe40a23480c47de1acb5c69cc3aa019d6f6cd36353cee85ac20a9e7?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/24fff15ecfe40a23480c47de1acb5c69cc3aa019d6f6cd36353cee85ac20a9e7?s=96&d=mm&r=g","caption":"James Cauchi"}}]}},"_links":{"self":[{"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/posts\/1362","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/comments?post=1362"}],"version-history":[{"count":47,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/posts\/1362\/revisions"}],"predecessor-version":[{"id":1675,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/posts\/1362\/revisions\/1675"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/media\/1679"}],"wp:attachment":[{"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/media?parent=1362"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/categories?post=1362"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/avinton.com\/en\/wp-json\/wp\/v2\/tags?post=1362"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}