001    package com.github.sarxos.webcam.ds.gstreamer;
002    
003    import java.awt.Dimension;
004    import java.awt.image.BufferedImage;
005    import java.awt.image.DataBufferInt;
006    import java.io.File;
007    import java.nio.IntBuffer;
008    import java.util.ArrayList;
009    import java.util.List;
010    import java.util.concurrent.TimeUnit;
011    import java.util.concurrent.atomic.AtomicBoolean;
012    
013    import org.bridj.Platform;
014    import org.gstreamer.Caps;
015    import org.gstreamer.Element;
016    import org.gstreamer.ElementFactory;
017    import org.gstreamer.Pad;
018    import org.gstreamer.Pipeline;
019    import org.gstreamer.State;
020    import org.gstreamer.Structure;
021    import org.gstreamer.elements.RGBDataSink;
022    import org.slf4j.Logger;
023    import org.slf4j.LoggerFactory;
024    
025    import com.github.sarxos.webcam.WebcamDevice;
026    import com.github.sarxos.webcam.WebcamResolution;
027    
028    
029    public class GStreamerDevice implements WebcamDevice, RGBDataSink.Listener, WebcamDevice.FPSSource {
030    
031            /**
032             * Logger.
033             */
034            private static final Logger LOG = LoggerFactory.getLogger(GStreamerDevice.class);
035    
036            /**
037             * Limit the lateness of frames to no more than 20ms (half a frame at 25fps)
038             */
039            private static final long LATENESS = 20; // ms
040    
041            /**
042             * Video format to capture.
043             */
044            private static final String FORMAT_MIME = "video/x-raw-yuv";
045    
046            /**
047             * All possible resolutions - populated while initialization phase.
048             */
049            private Dimension[] resolutions = null;
050    
051            /**
052             * Device name, immutable. Used only on Windows platform.
053             */
054            private final String name;
055            /**
056             * Device name, immutable. Used only on Linux platform.
057             */
058            private final File vfile;
059    
060            /* gstreamer stuff */
061    
062            private Pipeline pipe = null;
063            private Element source = null;
064            private Element filter = null;
065            private RGBDataSink sink = null;
066    
067            private Caps caps = null;
068    
069            /* logic */
070    
071            private AtomicBoolean open = new AtomicBoolean(false);
072            private AtomicBoolean disposed = new AtomicBoolean(false);
073            private AtomicBoolean starting = new AtomicBoolean(false);
074            private AtomicBoolean initialized = new AtomicBoolean(false);
075            private Dimension resolution = WebcamResolution.VGA.getSize();
076            private BufferedImage image = null;
077    
078            /* used to calculate fps */
079    
080            private long t1 = -1;
081            private long t2 = -1;
082    
083            private volatile double fps = 0;
084    
085            /**
086             * Create GStreamer webcam device.
087             * 
088             * @param name the name of webcam device
089             */
090            protected GStreamerDevice(String name) {
091                    this.name = name;
092                    this.vfile = null;
093            }
094    
095            protected GStreamerDevice(File vfile) {
096                    this.name = null;
097                    this.vfile = vfile;
098            }
099    
100            /**
101             * Initialize webcam device.
102             */
103            private synchronized void init() {
104    
105                    if (!initialized.compareAndSet(false, true)) {
106                            return;
107                    }
108    
109                    LOG.debug("GStreamer webcam device initialization");
110    
111                    pipe = new Pipeline(name);
112    
113                    if (Platform.isWindows()) {
114                            source = ElementFactory.make("dshowvideosrc", "source");
115                            source.set("device-name", name);
116                    } else if (Platform.isLinux()) {
117                            source = ElementFactory.make("v4l2src", "source");
118                            source.set("device", vfile.getAbsolutePath());
119                    }
120    
121                    sink = new RGBDataSink(name, this);
122                    sink.setPassDirectBuffer(true);
123                    sink.getSinkElement().setMaximumLateness(LATENESS, TimeUnit.MILLISECONDS);
124                    sink.getSinkElement().setQOSEnabled(true);
125    
126                    filter = ElementFactory.make("capsfilter", "filter");
127    
128                    if (Platform.isLinux()) {
129                            pipe.addMany(source, filter, sink);
130                            Element.linkMany(source, filter, sink);
131                            pipe.setState(State.READY);
132                    }
133    
134                    resolutions = parseResolutions(source.getPads().get(0));
135    
136                    if (Platform.isLinux()) {
137                            pipe.setState(State.NULL);
138                            Element.unlinkMany(source, filter, sink);
139                            pipe.removeMany(source, filter, sink);
140                    }
141            }
142    
143            /**
144             * Use GStreamer to get all possible resolutions.
145             * 
146             * @param pad the pad to get resolutions from
147             * @return Array of resolutions supported by device connected with pad
148             */
149            private static final Dimension[] parseResolutions(Pad pad) {
150    
151                    List<Dimension> dimensions = new ArrayList<Dimension>();
152    
153                    Caps caps = pad.getCaps();
154    
155                    Structure structure = null;
156                    String mime = null;
157    
158                    int n = caps.size();
159                    int i = 0;
160    
161                    int w = -1;
162                    int h = -1;
163    
164                    do {
165    
166                            structure = caps.getStructure(i++);
167    
168                            LOG.debug("Found format structure {}", structure);
169    
170                            mime = structure.getName();
171    
172                            if (mime.equals(FORMAT_MIME)) {
173                                    if (Platform.isWindows()) {
174                                            w = structure.getRange("width").getMinInt();
175                                            h = structure.getRange("height").getMinInt();
176                                            dimensions.add(new Dimension(w, h));
177                                    } else if (Platform.isLinux()) {
178                                            if ("YUY2".equals(structure.getFourccString("format"))) {
179                                                    w = structure.getInteger("width");
180                                                    h = structure.getInteger("height");
181                                                    dimensions.add(new Dimension(w, h));
182                                            }
183                                    }
184                            }
185    
186                    } while (i < n);
187    
188                    return dimensions.toArray(new Dimension[dimensions.size()]);
189            }
190    
191            @Override
192            public String getName() {
193                    if (Platform.isWindows()) {
194                            return name;
195                    } else if (Platform.isLinux()) {
196                            return vfile.getAbsolutePath();
197                    } else {
198                            throw new RuntimeException("Platform not supported by GStreamer capture driver");
199                    }
200            }
201    
202            @Override
203            public Dimension[] getResolutions() {
204                    init();
205                    return resolutions;
206            }
207    
208            @Override
209            public Dimension getResolution() {
210                    return resolution;
211            }
212    
213            @Override
214            public void setResolution(Dimension size) {
215                    this.resolution = size;
216            }
217    
218            @Override
219            public BufferedImage getImage() {
220                    return image;
221            }
222    
223            @Override
224            public void open() {
225    
226                    if (!open.compareAndSet(false, true)) {
227                            return;
228                    }
229    
230                    LOG.debug("Opening GStreamer device");
231    
232                    init();
233    
234                    starting.set(true);
235    
236                    Dimension size = getResolution();
237    
238                    image = new BufferedImage(size.width, size.height, BufferedImage.TYPE_INT_RGB);
239                    image.setAccelerationPriority(0);
240                    image.flush();
241    
242                    if (caps != null) {
243                            caps.dispose();
244                    }
245    
246                    caps = Caps.fromString(String.format("%s,width=%d,height=%d", FORMAT_MIME, size.width, size.height));
247    
248                    filter.setCaps(caps);
249    
250                    LOG.debug("Link elements");
251    
252                    pipe.addMany(source, filter, sink);
253                    Element.linkMany(source, filter, sink);
254                    pipe.setState(State.PLAYING);
255    
256                    // wait max 20s for image to appear
257                    synchronized (this) {
258                            LOG.debug("Wait for device to be ready");
259                            try {
260                                    this.wait(20000);
261                            } catch (InterruptedException e) {
262                                    return;
263                            }
264                    }
265            }
266    
267            @Override
268            public void close() {
269    
270                    if (!open.compareAndSet(true, false)) {
271                            return;
272                    }
273    
274                    LOG.debug("Closing GStreamer device");
275    
276                    image = null;
277    
278                    LOG.debug("Unlink elements");
279    
280                    pipe.setState(State.NULL);
281                    Element.unlinkMany(source, filter, sink);
282                    pipe.removeMany(source, filter, sink);
283            }
284    
285            @Override
286            public void dispose() {
287    
288                    if (!disposed.compareAndSet(false, true)) {
289                            return;
290                    }
291    
292                    LOG.debug("Disposing GStreamer device");
293    
294                    close();
295    
296                    filter.dispose();
297                    source.dispose();
298                    sink.dispose();
299                    pipe.dispose();
300                    caps.dispose();
301            }
302    
303            @Override
304            public boolean isOpen() {
305                    return open.get();
306            }
307    
308            @Override
309            public void rgbFrame(boolean preroll, int width, int height, IntBuffer rgb) {
310    
311                    LOG.trace("New RGB frame");
312    
313                    if (t1 == -1 || t2 == -1) {
314                            t1 = System.currentTimeMillis();
315                            t2 = System.currentTimeMillis();
316                    }
317    
318                    BufferedImage tmp = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
319                    tmp.setAccelerationPriority(0);
320                    tmp.flush();
321    
322                    rgb.get(((DataBufferInt) tmp.getRaster().getDataBuffer()).getData(), 0, width * height);
323    
324                    image = tmp;
325    
326                    if (starting.compareAndSet(true, false)) {
327    
328                            synchronized (this) {
329                                    this.notifyAll();
330                            }
331    
332                            LOG.debug("GStreamer device ready");
333                    }
334    
335                    t1 = t2;
336                    t2 = System.currentTimeMillis();
337    
338                    fps = (4 * fps + 1000 / (t2 - t1 + 1)) / 5;
339            }
340    
341            @Override
342            public double getFPS() {
343                    return fps;
344            }
345    
346            public Pipeline getPipe() {
347                    return pipe;
348            }
349    
350            public Element getSource() {
351                    return source;
352            }
353    
354            public Element getFilter() {
355                    return filter;
356            }
357    
358            public RGBDataSink getSink() {
359                    return sink;
360            }
361    
362            public Caps getCaps() {
363                    return caps;
364            }
365    }