Vielseitiges In-Memory Computing mit Apache Ignite und Kubernetes #ittage

Mit Apache Ignite steht eine hoch-performante, integrierte und verteilte In-Memory Plattform bereit die im Zusammenspiel mit Kubernetes zu wahrer Hochform aufläuft. In dieser Kombination lassen sich flexibel skalierbare In-Memory Computing Systeme elegant realisieren. In diesem Vortrag stellen wir die wesentlichen Features und die Architektur von Apache Ignite vor. Anhand von anschaulichen Beispielen zeigen wir mögliche Use-Cases, wie etwa den Einsatz als Kommunikations-Backbone einer Microservice-Architektur oder als Plattform zur Verarbeitung von kontinuierlichen Event-Daten. Zur Demonstration von Resilienz und Skalierbarkeit des In-Memory Data-Grids werden die Beispiele auf einem Kubernetes Cluster ausgeführt.

Transcript

9. 2015-04-04 2015-0 - 201 -0 - 201 -0 - 201 -0 - 201 - -
15. Ignition.setClientMode(true); Ignite ignite = Ignition.start("data-grid.xml"); CacheConfiguration<String, Company> cacheConfig = new CacheConfiguration<>("companyCache"); IgniteCache<String, Company> cache = ignite.getOrCreateCache(cacheConfig); Company qaware = new Company("1", "QAware GmbH", Long.MAX_VALUE); cache.putIfAbsent(qaware.getCompanyId(), qaware); SqlQuery<String, Company> sql = new SqlQuery<>(Company.class, "revenue > ?"); try (QueryCursor<Cache.Entry<String, Company>> cursor = cache.query(sql.setArgs(1_000_000))) { List<Company> companies = cursor.getAll().stream().map(Cache.Entry::getValue).collect(Collectors.toList()); ... } ignite.compute().affinityRun("companyCache", "1", () -> { Company company = cache.get("1"); // we collocated employees with the company, so access to the employee objects is local. IgniteCache<EmployeeKey, Employee> employeeCache = ignite.cache("employeeCache“); Employee reimer = employeeCache.localPeek(new EmployeeKey("11", company.getCompanyId())); ... });
17. Class.forName("org.apache.ignite.IgniteJdbcDriver"); Connection connection = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:10800?collocated=true"); Statement stmt = connection.createStatement()); stmt.executeUpdate("CREATE TABLE city (id LONG PRIMARY KEY, name VARCHAR) WITH \"template=partitioned, backups=0\""); stmt.executeUpdate("CREATE INDEX idx_city_name ON city (name)"); PreparedStatement stmt = connection.prepareStatement("INSERT INTO city (id, name) VALUES (?, ?)") stmt.setLong(1, 1L); stmt.setString(2, “Heidelberg"); stmt.executeUpdate(); Ignition.setClientMode(true); Ignite ignite = Ignition.start("ignite-sql.xml"); IgniteCache<Long, String> cityCache = ignite.cache("SQL_PUBLIC_CITY"); SqlFieldsQuery query = new SqlFieldsQuery("SELECT p.name, c.name FROM Person p, City c WHERE p.city_id = c.id"); FieldsQueryCursor<List<?>> cursor = cityCache.query(query); Iterator<List<?>> iterator = cursor.iterator(); SqlFieldsQuery query = new SqlFieldsQuery("DELETE FROM city"); cityCache.query(query).getAll();
18. <listener> <listener-class>...startup.servlet.ServletContextListenerStartup </listener-class> </listener> <filter> <filter-name>IgniteWebSessionsFilter</filter-name> <filter-class>...cache.websession.WebSessionFilter</filter-class> </filter> <filter-mapping> <filter-name>IgniteWebSessionsFilter</filter-name> <url-pattern>/*</url-pattern> </filter-mapping> <context-param> <param-name>IgniteConfigurationFilePath</param-name> <param-value>config/websession-config.xml</param-value> </context-param> <context-param> <param-name>IgniteWebSessionsCacheName</param-name> <param-value>partitioned</param-value> </context-param>
19. <hibernate-configuration> <session-factory> <property name="cache.use_second_level_cache">true</property> <property name="generate_statistics">true</property> <property name="cache.region.factory_class"> org.apache.ignite.cache.hibernate.HibernateRegionFactory </property> <property name="org.apache.ignite.hibernate.ignite_instance_name"> hibernate-ignite-grid </property> <property name="org.apache.ignite.hibernate.default_access_type"> TRANSACTIONAL </property> <mapping class="com.mycompany.MyEntity1"/> <class-cache class="com.mycompany.MyEntity1" usage="read-only"/> <collection-cache collection="com.mycompany.MyEntity1.children“ usage="read-only"/> </session-factory> </hibernate-configuration>
22. Ignite ignite = Ignition.start("compute-grid.xml"); IgniteCompute compute = ignite.compute(ignite.cluster().forRemotes()); compute.run(() -> System.out.println("Running distributed closure (1) on " + ignite.cluster().localNode().id())); compute.broadcast(() -> System.out.println("Broadcasting distributed closure on " + ignite.cluster().localNode().id())); final String question = "How many characters has this sentence?"; Collection<Integer> res = compute.apply(String::length, Arrays.asList(question.split(" "))); int total = res.stream().mapToInt(Integer::intValue).sum(); IgniteCache<Integer, String> cache = ignite.cache("someCache"); compute.affinityRun("someCache", key, () -> { System.out.println("Co-located processing [key= " + key + ", value= " + someCache.localPeek(key) + ']'); }); ExecutorService executorService = ignite.executorService(ignite.cluster().forRemotes()); executorService.submit(new IgniteRunnable() { @IgniteInstanceResource private Ignite ignite; @Override public void run() { System.out.println("Processing runnable on " + ignite.cluster().localNode().id() + " from grid job."); } });
25. Ignite ignite = Ignition.start("service-grid.xml"); IgniteServices services = ignite.services(); PingService pingService = services.serviceProxy("PingService", PingService.class, true); String pong = pingService.ping(); services.deployClusterSingleton("RandomUuidService", new DefaultRandomUuidService()); RandomUuidService randomUuidService = services.serviceProxy("RandomUuidService", RandomUuidService.class, false); String response = randomUuidService.randomUUID(); public class DefaultPingPongService implements PingService, Service { @IgniteInstanceResource private Ignite ignite; @Override public void init(ServiceContext ctx) throws Exception {...} @Override public void cancel(ServiceContext ctx) {...} @Override public void execute(ServiceContext ctx) throws Exception {...} @Override public String ping() { return "pong“; } }
28. Ignite ignite = Ignition.start("ignite-messaging.xml"); CollectionConfiguration colCfg = new CollectionConfiguration(); colCfg.setCacheMode(CacheMode.PARTITIONED); colCfg.setBackups(1); IgniteQueue<String> queue = ignite.queue("randomUuidQueue", 0, colCfg); queue.add(UUID.randomUUID().toString()); String uuid = queue.poll(5, TimeUnit.SECONDS); IgniteMessaging messaging = ignite.message(); messaging.sendOrdered("OrderedTopic", message, 5000L); messaging.send("UnorderedTopic", message); messaging.localListen("OrderedTopic", (nodeId, msg) -> { System.out.println("Received ordered message [msg=" + msg + ", from=" + nodeId + ']'); return true; // Return true to continue listening. }); messaging.localListen("UnorderedTopic", (nodeId, msg) -> { System.out.println("Received unordered message [msg=" + msg + ", from=" + nodeId + ']'); return true; // Return true to continue listening. });
31. IgniteDataStreamer<Long, String> dataStreamer = ignite.dataStreamer("tweetCache"); dataStreamer.allowOverwrite(true); dataStreamer.autoFlushFrequency(10); OAuthSettings oAuthSettings = new OAuthSettings(“consumerKey", “consumerSecret", “accessToken", “acessTokenSecret"); TwitterStreamer<Long, String> streamer = new TwitterStreamer<>(oAuthSettings); streamer.setIgnite(ignite); streamer.setStreamer(dataStreamer); final Gson gson = new Gson(); streamer.setSingleTupleExtractor(msg -> { Tweet tweet = gson.fromJson(msg, Tweet.class); return new GridMapEntry<>(tweet.id, tweet.text); }); Map<String, String> params = new HashMap<>(); params.put("track", "data2day,qaware,apache ignite,cloud"); params.put("follow", "1346627546,32837461,2650574588"); streamer.setApiParams(params); streamer.setEndpointUrl("/statuses/filter.json"); streamer.setThreadsCount(4); streamer.start();
34. echo "- The default provider is GCE" export KUBERNETES_PROVIDER=gce export KUBE_GCE_ZONE=europe-west1-d export NUM_NODES=4 echo "- Another possible provider is AWS" export KUBERNETES_PROVIDER=aws export KUBE_AWS_ZONE=eu-central-1a export NODE_SIZE=t2.small curl -sS https://get.k8s.io | bash
37. apiVersion: v1 kind: Service metadata: # Name of Ignite Service used by Kubernetes IP finder. # must be equal to TcpDiscoveryKubernetesIpFinder.serviceName. name: ignite spec: clusterIP: None # custom value. ports: - port: 9042 # custom value. selector: # Must be equal to one of the labels set in Ignite pods' # deployment configuration. app: ignite
38. apiVersion: v1 kind: ConfigMap metadata: name: kubernetes-ignite-xml data: kubernetes-ignite.xml: | <?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans ...> <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> ... <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. --> <property name="discoverySpi"> <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> <property name="ipFinder"> <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder"> </bean> </property> </bean> </property> </bean> </beans>
39. apiVersion: extensions/v1beta1 kind: Deployment metadata: name: ignite spec: replicas: 2 template: metadata: labels: app: ignite spec: containers: - name: ignite-node image: apacheignite/ignite:2.1.0 env: - name: CONFIG_URI value: file:///opt/ignite/apache-ignite-fabric-2.1.0-bin/data2day/config/kubernetes-ignite.xml - name: OPTION_LIBS value: ignite-kubernetes ports: ... volumeMounts: - mountPath: /opt/ignite/apache-ignite-fabric-2.1.0-bin/data2day/config name: ignite-config volumes: